From a6537b6bf1564ff1f8bb97c26f67a6718e2e884b Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Fri, 20 Dec 2024 17:02:28 +0000
Subject: [PATCH 001/112] added llava-next to gitingore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index ea20fe526..af4c8ea55 100755
--- a/.gitignore
+++ b/.gitignore
@@ -30,6 +30,7 @@ cache_dir
 ckpt
 pretrained/
 LLaVA/
+LLaVA-NeXT/
 *logs
 *.isorted
 temp/

From 315249d3a8ca0ce10cc576e9693b12227c5d6a4c Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Fri, 20 Dec 2024 17:03:03 +0000
Subject: [PATCH 002/112] init script for eval run

---
 run_eval.sh | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100755 run_eval.sh

diff --git a/run_eval.sh b/run_eval.sh
new file mode 100755
index 000000000..8709e5c66
--- /dev/null
+++ b/run_eval.sh
@@ -0,0 +1,22 @@
+out_root=/mnt/scratch-artemis/manos/data/tower-vision-eval-outputs
+
+mkdir -p $out_root/outs1/
+
+python3 -m accelerate.commands.launch \
+    --num_processes=1 \
+    -m lmms_eval \
+    --model molmo_hf \
+    --model_args pretrained='allenai/Molmo-7B-D-0924' \
+    --tasks mmmu,mme,scienceqa \
+    --batch_size 1 \
+    --log_samples \
+    --log_samples_suffix llava_next \
+    --output_path $out_root/outs1/
+
+    # --model_args pretrained="Unbabel/qwen2p5-7b-clip-hdr-sft-v3",add_system_prompt="Answer the questions." \
+    # --model_args pretrained="Unbabel/qwen2p5-7b-clip-hdr-sft-v3",add_system_prompt="Answer the questions." \
+    # --model_args pretrained="liuhaotian/llava-v1.5-7b"
+    #--model_args pretrained="lmms-lab/llama3-llava-next-8b",conv_template=llava_llama_3 \
+
+
+

From 4a1fe319485664dc8c92b6659faa8dc7d9282e88 Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Fri, 20 Dec 2024 17:03:24 +0000
Subject: [PATCH 003/112] init commit for adding molmo_hf

---
 lmms_eval/models/molmo_hf.py | 309 +++++++++++++++++++++++++++++++++++
 1 file changed, 309 insertions(+)
 create mode 100644 lmms_eval/models/molmo_hf.py

diff --git a/lmms_eval/models/molmo_hf.py b/lmms_eval/models/molmo_hf.py
new file mode 100644
index 000000000..434162bf9
--- /dev/null
+++ b/lmms_eval/models/molmo_hf.py
@@ -0,0 +1,309 @@
+import warnings
+from typing import List, Optional, Tuple, Union
+
+import numpy as np
+import PIL
+import torch
+from accelerate import Accelerator, DistributedType
+from accelerate.state import AcceleratorState
+from decord import VideoReader, cpu
+from tqdm import tqdm
+
+from transformers import AutoModelForCausalLM, AutoProcessor
+
+from lmms_eval import utils
+from lmms_eval.api.instance import Instance
+from lmms_eval.api.model import lmms
+from lmms_eval.api.registry import register_model
+
+warnings.filterwarnings("ignore")
+
+from loguru import logger as eval_logger
+
+DEFAULT_IMAGE_TOKEN = "<image>"
+DEFAULT_VIDEO_TOKEN = "<video>"
+
+
+
+
+@register_model("molmo_hf")
+class Molmo(lmms):
+    """
+    Molmo Model for Hugging Face Transformers: https://huggingface.co/collections/allenai/molmo-66f379e6fe3b8ef090a8ca19
+    Example usage:
+
+    accelerate launch --num_processes=8 --main_process_port 12345 -m lmms_eval \
+        --model molmo_hf \
+        --model_args pretrained=allenai/Molmo-7B-D-0924 \
+        --tasks seedbench \
+        --batch_size 1 \
+        --output_path ./logs/ \
+        --log_samples
+    """
+
+    def __init__(
+        self,
+        pretrained: str = "allenai/Molmo-7B-D-0924",
+        device: str = "cuda",
+        dtype: Optional[Union[str, torch.dtype]] = "auto",
+        batch_size: int = 1,
+        trust_remote_code: Optional[bool] = True,
+        device_map: str = "auto", # set to auto(?)
+        chat_template: Optional[str] = None,
+        use_cache: bool = True,
+        **kwargs,
+    ) -> None:
+        super().__init__()
+        # Do not use kwargs for now
+        assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
+
+        accelerator = Accelerator()
+        if accelerator.num_processes > 1 and device_map == "":
+            self._device = torch.device(f"cuda:{accelerator.local_process_index}")
+            self.device_map = f"cuda:{accelerator.local_process_index}"
+        else:
+            self._device = torch.device(device)
+            self.device_map = device_map
+        if isinstance(dtype, str) and dtype != "auto":
+            dtype = getattr(torch, dtype)
+
+        self._processor = AutoProcessor.from_pretrained(pretrained, trust_remote_code=trust_remote_code,device_map=device_map,torch_dtype=dtype)
+        # Pad from left for batched generation: https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/llava#usage-tips
+        #self._image_processor.tokenizer.padding_side = "left"
+
+        self._model = AutoModelForCausalLM.from_pretrained(pretrained, trust_remote_code=trust_remote_code, device_map=device_map, torch_dtype=dtype)
+        eval_logger.info(f"Using {type(self._model)} to instantiate the Molmo model.")
+        self.pretrained = pretrained
+        self._tokenizer = self._processor.tokenizer
+        self._config = self._model.config
+        import ipdb;ipdb.set_trace()
+        self.chat_template = chat_template
+
+        self.batch_size_per_gpu = int(batch_size)
+        self.use_cache = use_cache
+        
+        if accelerator.num_processes > 1 and device_map == "":
+            assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
+            # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
+            # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
+            # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
+            if accelerator.distributed_type == DistributedType.DEEPSPEED:
+                kwargs = {
+                    "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
+                    "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
+                }
+                AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
+                eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
+            if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
+                self._model = accelerator.prepare(self.model)
+            else:
+                self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
+            self.accelerator = accelerator
+            if self.accelerator.is_local_main_process:
+                eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
+            self._rank = self.accelerator.local_process_index
+            self._world_size = self.accelerator.num_processes
+        elif accelerator.num_processes == 1 and device_map == "auto":
+            eval_logger.info(f"Using {accelerator.num_processes} devices with pipeline parallelism")
+            self._rank = 0
+            self._word_size = 1
+        else:
+            eval_logger.info(f"Using single device: {self._device}")
+            self.model.to(self._device)
+            self._rank = 0
+            self._word_size = 1
+        self.accelerator = accelerator
+
+    @property
+    def config(self):
+        # return the associated transformers.AutoConfig for the given pretrained model.
+        return self._config
+
+    @property
+    def tokenizer(self):
+        return self._tokenizer
+
+    @property
+    def model(self):
+        # returns the model, unwrapping it if using Accelerate
+        if hasattr(self, "accelerator"):
+            return self.accelerator.unwrap_model(self._model)
+        else:
+            return self._model
+
+    @property
+    def eot_token_id(self):
+        # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
+        return self._tokenizer.eos_token_id
+
+    @property
+    def max_length(self):
+        return self._max_length
+
+    @property
+    def batch_size(self):
+        return self.batch_size_per_gpu
+
+    @property
+    def device(self):
+        return self._device
+
+    @property
+    def rank(self):
+        return self._rank
+
+    @property
+    def world_size(self):
+        return self._world_size
+
+    def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
+        """ """
+        add_special_tokens = False if add_special_tokens is None else add_special_tokens
+        encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
+        # left-truncate the encoded context to be at most `left_truncate_len` tokens long
+        if left_truncate_len:
+            encoding = encoding[-left_truncate_len:]
+        return encoding
+
+    def tok_decode(self, tokens):
+        return self.tokenizer.decode(tokens)
+
+    def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
+        raise NotImplementedError("Loglikelihood not implemented for Molmo")
+
+    def flatten(self, input):
+        new_list = []
+        for i in input:
+            for j in i:
+                new_list.append(j)
+        return new_list
+
+    
+
+    def generate_until(self, requests: List[Instance]) -> List[str]:
+        res = []
+
+        def _collate(x):
+            # the negative sign on len(toks) sorts descending - this has a few advantages:
+            # - time estimates will always be over not underestimates, which is more useful for planning
+            # - to know the size of a batch when going through the list, you know the first one is always the batch
+            #   padded context length. this is useful to simplify the batching logic and more importantly to make
+            #   automatic adaptive batches much much easier to implement
+            # - any OOMs will happen right away rather than near the end
+            toks = self.tok_encode(x[0])
+            return -len(toks), x[0]
+
+        # we group requests by their generation_kwargs,
+        # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
+        # in the same batch.
+        re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
+        chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
+        num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
+        pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
+        for chunk in chunks:
+            contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
+            task = task[0]
+            split = split[0]
+            visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
+            visuals = self.flatten(visuals)
+            if len(visuals) == 0:
+                task_type = "text"
+            elif isinstance(visuals[0], PIL.Image.Image):
+                task_type = "image"
+            elif isinstance(visuals[0], str):
+                task_type = "video"
+            # we assume all gen kwargs in the batch are the same
+            # this is safe to assume because the `grouper` object ensures it.
+            gen_kwargs = all_gen_kwargs[0]
+
+            # Set default values for until and max_new_tokens
+            until = [self.tok_decode(self.eot_token_id)]
+
+            # Update values from gen_kwargs if present
+            if "until" in gen_kwargs:
+                until = gen_kwargs.pop("until")
+                if isinstance(until, str):
+                    until = [until]
+                elif not isinstance(until, list):
+                    raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
+            assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
+            context = contexts[0]
+
+            # Some benchmarks like MME do not contain image tokens, so we prepend them to the prompt.
+            if DEFAULT_IMAGE_TOKEN not in context:
+                if task_type == "image":
+                    image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visuals)
+                elif task_type == "video":
+                    image_tokens = [DEFAULT_VIDEO_TOKEN] * len(visuals)
+                image_tokens = " ".join(image_tokens)
+                context = f"{image_tokens}\n{context}"
+            # Apply chat template
+            messages = [{"role": "user", "content": context}]
+            if self.chat_template is not None:
+                self.tokenizer.chat_template = self.chat_template
+                text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+            elif self.tokenizer.chat_template is not None:
+                text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+            else:
+                self.tokenizer.chat_template = VICUNA_CHAT_TEMPLATE
+                text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+
+            if self.accelerator.is_main_process and doc_id[0] % 100 == 0:
+                eval_logger.debug(f"Prompt for doc ID {doc_id[0]}:\n\n{text}\n")
+
+            if task_type == "video":
+                try:
+                    visuals = [self.load_video(visuals, self.max_frames_num)]
+                except Exception as e:
+                    res.append("")
+                    eval_logger.info(f"Error {e} when loading video : {visuals}")
+                    pbar.update(1)
+
+            if task_type == "image":
+                inputs = self._image_processor(images=visuals, text=text, return_tensors="pt").to(self._device, self.model.dtype)
+            elif task_type == "video":
+                inputs = self._image_processor(videos=visuals, text=text, return_tensors="pt").to(self._device, self.model.dtype)
+
+            gen_kwargs["image_sizes"] = [visuals[idx].size for idx in range(len(visuals))]
+            if "max_new_tokens" not in gen_kwargs:
+                gen_kwargs["max_new_tokens"] = 1024
+            if "temperature" not in gen_kwargs:
+                gen_kwargs["temperature"] = 0
+            if "top_p" not in gen_kwargs:
+                gen_kwargs["top_p"] = None
+            if "num_beams" not in gen_kwargs:
+                gen_kwargs["num_beams"] = 1
+            try:
+                cont = self.model.generate(
+                    **inputs,
+                    do_sample=True if gen_kwargs["temperature"] > 0 else False,
+                    temperature=gen_kwargs["temperature"],
+                    top_p=gen_kwargs["top_p"],
+                    num_beams=gen_kwargs["num_beams"],
+                    max_new_tokens=gen_kwargs["max_new_tokens"],
+                    use_cache=self.use_cache,
+                    pad_token_id=self.eot_token_id,
+                    eos_token_id=self.eot_token_id,
+                )
+                cont = cont[:, inputs["input_ids"].shape[-1] :]
+            except Exception as e:
+                eval_logger.error(f"Error {e} in generating")
+                cont = ""
+            text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)[0]
+            if self.accelerator.is_main_process and doc_id[0] % 100 == 0:
+                eval_logger.debug(f"Generated text for doc ID {doc_id[0]}:\n\n{text_outputs}\n")
+
+            res.append(text_outputs)
+            self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
+            pbar.update(1)
+        # reorder this group of results back to original unsorted form
+        res = re_ords.get_original(res)
+
+        pbar.close()
+        return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("Not implemented for Molmo!!")
+    
+    def load_video(self, video_path, max_frames_num):
+        raise NotImplementedError("Not implemented for Molmo!!")

From 03b46e704f23c5b0da282eb784febf227d86b12a Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Fri, 20 Dec 2024 17:04:06 +0000
Subject: [PATCH 004/112] requirements commit -- to be MODIFIED

---
 lmms-eval.reqs       | 185 +++++++++++++++++++++++++++++++++++++++++++
 make_requirements.py |  23 ++++++
 requirements.txt     | 183 ++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 391 insertions(+)
 create mode 100644 lmms-eval.reqs
 create mode 100644 make_requirements.py
 create mode 100644 requirements.txt

diff --git a/lmms-eval.reqs b/lmms-eval.reqs
new file mode 100644
index 000000000..6830faa7d
--- /dev/null
+++ b/lmms-eval.reqs
@@ -0,0 +1,185 @@
+accelerate                    1.2.0
+aiofiles                      23.2.1
+aiohappyeyeballs              2.4.4
+aiohttp                       3.11.10
+aiosignal                     1.3.1
+altair                        5.5.0
+annotated-types               0.7.0
+anyio                         4.7.0
+async-timeout                 5.0.1
+attrs                         24.2.0
+av                            14.0.1
+bitsandbytes                  0.45.0
+black                         24.1.0
+certifi                       2024.8.30
+cfgv                          3.4.0
+chardet                       5.2.0
+charset-normalizer            3.4.0
+click                         8.1.7
+colorama                      0.4.6
+contourpy                     1.3.1
+cycler                        0.12.1
+DataProperty                  1.0.1
+datasets                      2.20.0
+decord                        0.6.0
+dill                          0.3.7
+distlib                       0.3.9
+distro                        1.9.0
+docker-pycreds                0.4.0
+einops                        0.6.1
+einops-exts                   0.0.4
+et_xmlfile                    2.0.0
+evaluate                      0.4.3
+exceptiongroup                1.2.2
+fastapi                       0.115.6
+ffmpy                         0.4.0
+filelock                      3.16.1
+fonttools                     4.55.2
+frozenlist                    1.5.0
+fsspec                        2023.10.0
+ftfy                          6.3.1
+gitdb                         4.0.11
+GitPython                     3.1.43
+gradio                        4.16.0
+gradio_client                 0.8.1
+h11                           0.14.0
+hf_transfer                   0.1.8
+httpcore                      0.16.3
+httpx                         0.24.0
+huggingface-hub               0.26.5
+identify                      2.6.3
+idna                          3.10
+importlib_resources           6.4.5
+isort                         5.13.2
+Jinja2                        3.1.4
+jiter                         0.8.2
+joblib                        1.4.2
+jsonlines                     4.0.0
+jsonschema                    4.23.0
+jsonschema-specifications     2024.10.1
+kiwisolver                    1.4.7
+latex2mathml                  3.77.0
+llava                         1.2.2.post1 /lustre/fshomisc/home/rech/genrce01/ued79zb/repos/LLaVA
+lmms_eval                     0.3.0       /lustre/fshomisc/home/rech/genrce01/ued79zb/repos/lmms-eval
+loguru                        0.7.3
+lxml                          5.3.0
+markdown-it-py                3.0.0
+markdown2                     2.5.1
+MarkupSafe                    2.1.5
+matplotlib                    3.9.3
+mbstrdecoder                  1.1.3
+mdurl                         0.1.2
+mpmath                        1.3.0
+multidict                     6.1.0
+multiprocess                  0.70.15
+mypy-extensions               1.0.0
+narwhals                      1.17.0
+networkx                      3.4.2
+nltk                          3.9.1
+nodeenv                       1.9.1
+numexpr                       2.10.2
+numpy                         1.26.4
+nvidia-cublas-cu12            12.1.3.1
+nvidia-cuda-cupti-cu12        12.1.105
+nvidia-cuda-nvrtc-cu12        12.1.105
+nvidia-cuda-runtime-cu12      12.1.105
+nvidia-cudnn-cu12             8.9.2.26
+nvidia-cufft-cu12             11.0.2.54
+nvidia-curand-cu12            10.3.2.106
+nvidia-cusolver-cu12          11.4.5.107
+nvidia-cusparse-cu12          12.1.0.106
+nvidia-nccl-cu12              2.18.1
+nvidia-nvjitlink-cu12         12.4.127
+nvidia-nvtx-cu12              12.1.105
+openai                        1.57.2
+opencv-python-headless        4.10.0.84
+openpyxl                      3.1.5
+orjson                        3.10.12
+packaging                     24.2
+pandas                        2.2.3
+pathspec                      0.12.1
+pathvalidate                  3.2.1
+peft                          0.14.0
+pillow                        10.4.0
+pip                           24.2
+platformdirs                  4.3.6
+portalocker                   3.0.0
+pre_commit                    4.0.1
+propcache                     0.2.1
+protobuf                      3.20.0
+psutil                        6.1.0
+pyarrow                       18.1.0
+pyarrow-hotfix                0.6
+pybind11                      2.13.6
+pycocoevalcap                 1.2
+pycocotools                   2.0.8
+pydantic                      2.10.3
+pydantic_core                 2.27.1
+pydub                         0.25.1
+Pygments                      2.18.0
+pyparsing                     3.2.0
+pytablewriter                 1.2.0
+python-dateutil               2.9.0.post0
+python-multipart              0.0.19
+pytz                          2024.2
+PyYAML                        6.0.2
+referencing                   0.35.1
+regex                         2024.11.6
+requests                      2.32.3
+rfc3986                       1.5.0
+rich                          13.9.4
+rpds-py                       0.22.3
+ruff                          0.8.2
+sacrebleu                     2.4.3
+safetensors                   0.4.5
+scikit-learn                  1.2.2
+scipy                         1.14.1
+semantic-version              2.10.0
+sentence-transformers         3.3.1
+sentencepiece                 0.1.99
+sentry-sdk                    2.19.2
+setproctitle                  1.3.4
+setuptools                    75.1.0
+shellingham                   1.5.4
+shortuuid                     1.0.13
+six                           1.17.0
+smmap                         5.0.1
+sniffio                       1.3.1
+sqlitedict                    2.1.0
+starlette                     0.41.3
+svgwrite                      1.4.3
+sympy                         1.13.1
+tabledata                     1.3.3
+tabulate                      0.9.0
+tcolorpy                      0.1.6
+tenacity                      8.3.0
+threadpoolctl                 3.5.0
+tiktoken                      0.8.0
+timm                          0.6.13
+tokenizers                    0.21.0
+tomli                         2.2.1
+tomlkit                       0.12.0
+torch                         2.1.2
+torchvision                   0.16.2
+tqdm                          4.67.1
+tqdm-multiprocess             0.0.11
+transformers                  4.47.0
+transformers-stream-generator 0.0.5
+triton                        2.1.0
+typepy                        1.3.2
+typer                         0.15.1
+typing_extensions             4.12.2
+tzdata                        2024.2
+urllib3                       2.2.3
+uvicorn                       0.32.1
+virtualenv                    20.28.0
+wandb                         0.19.0
+wavedrom                      2.0.3.post3
+wcwidth                       0.2.13
+websockets                    11.0.3
+wheel                         0.44.0
+xxhash                        3.5.0
+yarl                          1.18.3
+yt-dlp                        2024.12.6
+zss                           1.2.0
+zstandard                     0.23.0
\ No newline at end of file
diff --git a/make_requirements.py b/make_requirements.py
new file mode 100644
index 000000000..faa0520c6
--- /dev/null
+++ b/make_requirements.py
@@ -0,0 +1,23 @@
+import re
+
+# Input file name containing the dependencies (or replace this with direct input text)
+input_file = "./lmms-eval.reqs"
+output_file = "./requirements.txt"
+
+def clean_dependencies(input_file, output_file):
+    with open(input_file, "r") as infile, open(output_file, "w") as outfile:
+        for line in infile:
+            # Skip empty lines or lines with local paths
+            if not line.strip():
+                continue
+            
+            # Match lines with valid package specifications
+            match = re.match(r"^(\S+)\s+(\S+)(?!.*\s+/\S+)$", line)
+            if match:
+                package, version = match.groups()
+                outfile.write(f"{package}=={version}\n")
+
+# Run the function to clean and generate requirements.txt
+clean_dependencies(input_file, output_file)
+
+print(f"Cleaned requirements saved to {output_file}.")
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 000000000..11e2127ac
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,183 @@
+accelerate==1.2.0
+aiofiles==23.2.1
+aiohappyeyeballs==2.4.4
+aiohttp==3.11.10
+aiosignal==1.3.1
+altair==5.5.0
+annotated-types==0.7.0
+anyio==4.7.0
+async-timeout==5.0.1
+attrs==24.2.0
+av==14.0.1
+bitsandbytes==0.45.0
+black==24.1.0
+certifi==2024.8.30
+cfgv==3.4.0
+chardet==5.2.0
+charset-normalizer==3.4.0
+click==8.1.7
+colorama==0.4.6
+contourpy==1.3.1
+cycler==0.12.1
+DataProperty==1.0.1
+datasets==2.20.0
+decord==0.6.0
+dill==0.3.7
+distlib==0.3.9
+distro==1.9.0
+docker-pycreds==0.4.0
+einops==0.6.1
+einops-exts==0.0.4
+et_xmlfile==2.0.0
+evaluate==0.4.3
+exceptiongroup==1.2.2
+fastapi==0.115.6
+ffmpy==0.4.0
+filelock==3.16.1
+fonttools==4.55.2
+frozenlist==1.5.0
+fsspec==2023.10.0
+ftfy==6.3.1
+gitdb==4.0.11
+GitPython==3.1.43
+gradio==4.16.0
+gradio_client==0.8.1
+h11==0.14.0
+hf_transfer==0.1.8
+httpcore==0.16.3
+httpx==0.24.0
+huggingface-hub==0.26.5
+identify==2.6.3
+idna==3.10
+importlib_resources==6.4.5
+isort==5.13.2
+Jinja2==3.1.4
+jiter==0.8.2
+joblib==1.4.2
+jsonlines==4.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+kiwisolver==1.4.7
+latex2mathml==3.77.0
+loguru==0.7.3
+lxml==5.3.0
+markdown-it-py==3.0.0
+markdown2==2.5.1
+MarkupSafe==2.1.5
+matplotlib==3.9.3
+mbstrdecoder==1.1.3
+mdurl==0.1.2
+mpmath==1.3.0
+multidict==6.1.0
+multiprocess==0.70.15
+mypy-extensions==1.0.0
+narwhals==1.17.0
+networkx==3.4.2
+nltk==3.9.1
+nodeenv==1.9.1
+numexpr==2.10.2
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.18.1
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nvtx-cu12==12.1.105
+openai==1.57.2
+opencv-python-headless==4.10.0.84
+openpyxl==3.1.5
+orjson==3.10.12
+packaging==24.2
+pandas==2.2.3
+pathspec==0.12.1
+pathvalidate==3.2.1
+peft==0.14.0
+pillow==10.4.0
+pip==24.2
+platformdirs==4.3.6
+portalocker==3.0.0
+pre_commit==4.0.1
+propcache==0.2.1
+protobuf==3.20.0
+psutil==6.1.0
+pyarrow==18.1.0
+pyarrow-hotfix==0.6
+pybind11==2.13.6
+pycocoevalcap==1.2
+pycocotools==2.0.8
+pydantic==2.10.3
+pydantic_core==2.27.1
+pydub==0.25.1
+Pygments==2.18.0
+pyparsing==3.2.0
+pytablewriter==1.2.0
+python-dateutil==2.9.0.post0
+python-multipart==0.0.19
+pytz==2024.2
+PyYAML==6.0.2
+referencing==0.35.1
+regex==2024.11.6
+requests==2.32.3
+rfc3986==1.5.0
+rich==13.9.4
+rpds-py==0.22.3
+ruff==0.8.2
+sacrebleu==2.4.3
+safetensors==0.4.5
+scikit-learn==1.2.2
+scipy==1.14.1
+semantic-version==2.10.0
+sentence-transformers==3.3.1
+sentencepiece==0.1.99
+sentry-sdk==2.19.2
+setproctitle==1.3.4
+setuptools==75.1.0
+shellingham==1.5.4
+shortuuid==1.0.13
+six==1.17.0
+smmap==5.0.1
+sniffio==1.3.1
+sqlitedict==2.1.0
+starlette==0.41.3
+svgwrite==1.4.3
+sympy==1.13.1
+tabledata==1.3.3
+tabulate==0.9.0
+tcolorpy==0.1.6
+tenacity==8.3.0
+threadpoolctl==3.5.0
+tiktoken==0.8.0
+timm==0.6.13
+tokenizers==0.21.0
+tomli==2.2.1
+tomlkit==0.12.0
+torch==2.1.2
+torchvision==0.16.2
+tqdm==4.67.1
+tqdm-multiprocess==0.0.11
+transformers==4.47.0
+transformers-stream-generator==0.0.5
+triton==2.1.0
+typepy==1.3.2
+typer==0.15.1
+typing_extensions==4.12.2
+tzdata==2024.2
+urllib3==2.2.3
+uvicorn==0.32.1
+virtualenv==20.28.0
+wandb==0.19.0
+wavedrom==2.0.3.post3
+wcwidth==0.2.13
+websockets==11.0.3
+wheel==0.44.0
+xxhash==3.5.0
+yarl==1.18.3
+yt-dlp==2024.12.6
+zss==1.2.0
+zstandard==0.23.0

From a4c8794e568fc70be1c40d2e7011c10646c740ab Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Mon, 30 Dec 2024 15:10:40 +0000
Subject: [PATCH 005/112] addition of pangea tasks+ pangea model+ script for
 aggregating results

---
 lmms_eval/models/__init__.py                  |    1 +
 lmms_eval/models/pangea.py                    |  429 +++++
 lmms_eval/tasks/cvqa/cvqa.yaml                |   31 +
 lmms_eval/tasks/cvqa/cvqa_translated.yaml     |   31 +
 lmms_eval/tasks/cvqa/utils.py                 |   90 ++
 lmms_eval/tasks/m3exam/m3exam.yaml            |    9 +
 lmms_eval/tasks/m3exam/m3exam_afrikaans.yaml  |   20 +
 lmms_eval/tasks/m3exam/m3exam_chinese.yaml    |   20 +
 lmms_eval/tasks/m3exam/m3exam_english.yaml    |   20 +
 lmms_eval/tasks/m3exam/m3exam_italian.yaml    |   20 +
 lmms_eval/tasks/m3exam/m3exam_portuguese.yaml |   20 +
 lmms_eval/tasks/m3exam/m3exam_thai.yaml       |   20 +
 lmms_eval/tasks/m3exam/m3exam_vietnamese.yaml |   20 +
 lmms_eval/tasks/m3exam/utils.py               |  242 +++
 lmms_eval/tasks/marvl/marvl.yaml              |    8 +
 lmms_eval/tasks/marvl/marvl_id.yaml           |   30 +
 lmms_eval/tasks/marvl/marvl_sw.yaml           |   31 +
 lmms_eval/tasks/marvl/marvl_ta.yaml           |   31 +
 lmms_eval/tasks/marvl/marvl_tr.yaml           |   31 +
 lmms_eval/tasks/marvl/marvl_zh.yaml           |   31 +
 lmms_eval/tasks/marvl/nlvr2.yaml              |   30 +
 lmms_eval/tasks/marvl/utils.py                |   25 +
 lmms_eval/tasks/maxm/evaluation.py            | 1382 +++++++++++++++++
 lmms_eval/tasks/maxm/maxm.yaml                |    9 +
 lmms_eval/tasks/maxm/maxm_en.yaml             |   57 +
 lmms_eval/tasks/maxm/maxm_fr.yaml             |   57 +
 lmms_eval/tasks/maxm/maxm_hi.yaml             |   57 +
 lmms_eval/tasks/maxm/maxm_iw.yaml             |   57 +
 lmms_eval/tasks/maxm/maxm_ro.yaml             |   57 +
 lmms_eval/tasks/maxm/maxm_th.yaml             |   57 +
 lmms_eval/tasks/maxm/maxm_zh.yaml             |   57 +
 lmms_eval/tasks/maxm/utils.py                 |  566 +++++++
 lmms_eval/tasks/xgqa/utils.py                 |   18 +
 lmms_eval/tasks/xgqa/xgqa.yaml                |   10 +
 lmms_eval/tasks/xgqa/xgqa_bn.yaml             |   33 +
 lmms_eval/tasks/xgqa/xgqa_de.yaml             |   33 +
 lmms_eval/tasks/xgqa/xgqa_en.yaml             |   33 +
 lmms_eval/tasks/xgqa/xgqa_id.yaml             |   33 +
 lmms_eval/tasks/xgqa/xgqa_ko.yaml             |   33 +
 lmms_eval/tasks/xgqa/xgqa_pt.yaml             |   33 +
 lmms_eval/tasks/xgqa/xgqa_ru.yaml             |   33 +
 lmms_eval/tasks/xgqa/xgqa_zh.yaml             |   33 +
 lmms_eval/tasks/xm100/utils.py                |  202 +++
 lmms_eval/tasks/xm100/xm100.yaml              |   38 +
 lmms_eval/tasks/xm100/xm100_ar.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_bn.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_cs.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_da.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_de.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_el.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_en.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_es.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_fa.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_fi.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_fil.yaml          |   42 +
 lmms_eval/tasks/xm100/xm100_fr.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_he.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_hi.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_hr.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_hu.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_id.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_it.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_ja.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_ko.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_mi.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_nl.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_no.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_pl.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_pt.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_quz.yaml          |   42 +
 lmms_eval/tasks/xm100/xm100_ro.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_ru.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_sv.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_sw.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_te.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_th.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_tr.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_uk.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_vi.yaml           |   42 +
 lmms_eval/tasks/xm100/xm100_zh.yaml           |   42 +
 lmms_eval/tasks/xmmmu/mmmu_Arabic_val.yaml    |   21 +
 lmms_eval/tasks/xmmmu/mmmu_English_val.yaml   |   21 +
 lmms_eval/tasks/xmmmu/mmmu_French_val.yaml    |   21 +
 lmms_eval/tasks/xmmmu/mmmu_Hindi_val.yaml     |   21 +
 .../tasks/xmmmu/mmmu_Indonesian_val.yaml      |   21 +
 lmms_eval/tasks/xmmmu/mmmu_Japanese_val.yaml  |   21 +
 .../tasks/xmmmu/mmmu_Portuguese_val.yaml      |   21 +
 lmms_eval/tasks/xmmmu/utils.py                |  480 ++++++
 lmms_eval/tasks/xmmmu/xmmmu.yaml              |   13 +
 run_eval.sh                                   |   50 +-
 run_eval_aggregate.py                         |  100 ++
 91 files changed, 6371 insertions(+), 9 deletions(-)
 create mode 100644 lmms_eval/models/pangea.py
 create mode 100644 lmms_eval/tasks/cvqa/cvqa.yaml
 create mode 100644 lmms_eval/tasks/cvqa/cvqa_translated.yaml
 create mode 100644 lmms_eval/tasks/cvqa/utils.py
 create mode 100644 lmms_eval/tasks/m3exam/m3exam.yaml
 create mode 100644 lmms_eval/tasks/m3exam/m3exam_afrikaans.yaml
 create mode 100644 lmms_eval/tasks/m3exam/m3exam_chinese.yaml
 create mode 100644 lmms_eval/tasks/m3exam/m3exam_english.yaml
 create mode 100644 lmms_eval/tasks/m3exam/m3exam_italian.yaml
 create mode 100644 lmms_eval/tasks/m3exam/m3exam_portuguese.yaml
 create mode 100644 lmms_eval/tasks/m3exam/m3exam_thai.yaml
 create mode 100644 lmms_eval/tasks/m3exam/m3exam_vietnamese.yaml
 create mode 100644 lmms_eval/tasks/m3exam/utils.py
 create mode 100755 lmms_eval/tasks/marvl/marvl.yaml
 create mode 100755 lmms_eval/tasks/marvl/marvl_id.yaml
 create mode 100755 lmms_eval/tasks/marvl/marvl_sw.yaml
 create mode 100755 lmms_eval/tasks/marvl/marvl_ta.yaml
 create mode 100755 lmms_eval/tasks/marvl/marvl_tr.yaml
 create mode 100755 lmms_eval/tasks/marvl/marvl_zh.yaml
 create mode 100755 lmms_eval/tasks/marvl/nlvr2.yaml
 create mode 100755 lmms_eval/tasks/marvl/utils.py
 create mode 100644 lmms_eval/tasks/maxm/evaluation.py
 create mode 100755 lmms_eval/tasks/maxm/maxm.yaml
 create mode 100755 lmms_eval/tasks/maxm/maxm_en.yaml
 create mode 100755 lmms_eval/tasks/maxm/maxm_fr.yaml
 create mode 100755 lmms_eval/tasks/maxm/maxm_hi.yaml
 create mode 100755 lmms_eval/tasks/maxm/maxm_iw.yaml
 create mode 100755 lmms_eval/tasks/maxm/maxm_ro.yaml
 create mode 100755 lmms_eval/tasks/maxm/maxm_th.yaml
 create mode 100755 lmms_eval/tasks/maxm/maxm_zh.yaml
 create mode 100755 lmms_eval/tasks/maxm/utils.py
 create mode 100755 lmms_eval/tasks/xgqa/utils.py
 create mode 100755 lmms_eval/tasks/xgqa/xgqa.yaml
 create mode 100755 lmms_eval/tasks/xgqa/xgqa_bn.yaml
 create mode 100755 lmms_eval/tasks/xgqa/xgqa_de.yaml
 create mode 100755 lmms_eval/tasks/xgqa/xgqa_en.yaml
 create mode 100755 lmms_eval/tasks/xgqa/xgqa_id.yaml
 create mode 100755 lmms_eval/tasks/xgqa/xgqa_ko.yaml
 create mode 100755 lmms_eval/tasks/xgqa/xgqa_pt.yaml
 create mode 100755 lmms_eval/tasks/xgqa/xgqa_ru.yaml
 create mode 100755 lmms_eval/tasks/xgqa/xgqa_zh.yaml
 create mode 100644 lmms_eval/tasks/xm100/utils.py
 create mode 100644 lmms_eval/tasks/xm100/xm100.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_ar.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_bn.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_cs.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_da.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_de.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_el.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_en.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_es.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_fa.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_fi.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_fil.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_fr.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_he.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_hi.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_hr.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_hu.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_id.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_it.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_ja.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_ko.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_mi.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_nl.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_no.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_pl.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_pt.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_quz.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_ro.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_ru.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_sv.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_sw.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_te.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_th.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_tr.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_uk.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_vi.yaml
 create mode 100644 lmms_eval/tasks/xm100/xm100_zh.yaml
 create mode 100644 lmms_eval/tasks/xmmmu/mmmu_Arabic_val.yaml
 create mode 100644 lmms_eval/tasks/xmmmu/mmmu_English_val.yaml
 create mode 100644 lmms_eval/tasks/xmmmu/mmmu_French_val.yaml
 create mode 100644 lmms_eval/tasks/xmmmu/mmmu_Hindi_val.yaml
 create mode 100644 lmms_eval/tasks/xmmmu/mmmu_Indonesian_val.yaml
 create mode 100644 lmms_eval/tasks/xmmmu/mmmu_Japanese_val.yaml
 create mode 100644 lmms_eval/tasks/xmmmu/mmmu_Portuguese_val.yaml
 create mode 100644 lmms_eval/tasks/xmmmu/utils.py
 create mode 100644 lmms_eval/tasks/xmmmu/xmmmu.yaml
 create mode 100644 run_eval_aggregate.py

diff --git a/lmms_eval/models/__init__.py b/lmms_eval/models/__init__.py
index 76deee2e1..a2082d34f 100644
--- a/lmms_eval/models/__init__.py
+++ b/lmms_eval/models/__init__.py
@@ -56,6 +56,7 @@
     "videochat2": "VideoChat2",
     "llama_vision": "LlamaVision",
     "aria": "Aria",
+    "pangea": "Pangea",
 }
 
 
diff --git a/lmms_eval/models/pangea.py b/lmms_eval/models/pangea.py
new file mode 100644
index 000000000..768fc87eb
--- /dev/null
+++ b/lmms_eval/models/pangea.py
@@ -0,0 +1,429 @@
+import torch
+
+torch.backends.cuda.matmul.allow_tf32 = True
+
+
+import copy
+from tqdm import tqdm
+from datetime import timedelta
+
+from lmms_eval import utils
+from lmms_eval.api.instance import Instance
+from lmms_eval.api.model import lmms
+from lmms_eval.api.registry import register_model
+from lmms_eval.utils import stop_sequences_criteria
+
+from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
+from accelerate.state import AcceleratorState
+from typing import List, Optional, Union, Tuple
+from packaging import version
+import warnings
+
+warnings.filterwarnings("ignore")
+
+from loguru import logger as eval_logger
+
+try:
+    from llava.model.builder import load_pretrained_model
+    from llava.mm_utils import get_model_name_from_path, process_images, tokenizer_image_token
+    from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN
+    from llava.conversation import conv_templates
+except Exception as e:
+    eval_logger.debug("LLaVA is not installed. Please install LLaVA to use this model.\nError: %s" % e)
+
+# inference implementation for attention, can be "sdpa", "eager", "flash_attention_2". Seems FA2 is not effective during inference: https://discuss.huggingface.co/t/flash-attention-has-no-effect-on-inference/73453/5
+# if is_flash_attn_2_available:
+#     best_fit_attn_implementation = "flash_attention_2" # flash_attn has a bug that says: ERROR Error query and key must have the same dtype in generating
+
+if version.parse(torch.__version__) >= version.parse("2.1.2"):
+    best_fit_attn_implementation = "sdpa"
+else:
+    best_fit_attn_implementation = "eager"
+
+
+@register_model("pangea")
+class Pangea(lmms):
+    """
+    Pangea Model
+    """
+
+    def __init__(
+        self,
+        pretrained: str = "neulab/Pangea-7B",
+        truncation: Optional[bool] = True,
+        device: Optional[str] = "cuda:0",
+        batch_size: Optional[Union[int, str]] = 1,
+        model_name="pangea-7b-llava-clip-qwen2",
+        attn_implementation=best_fit_attn_implementation,
+        device_map="cuda:0",
+        conv_template="qwen_1_5",
+        use_cache=True,
+        tie_weights: bool = True,
+        truncate_context=False,  # whether to truncate the context in generation, set it False for LLaVA-1.6
+        customized_config=None,  # ends in json
+        **kwargs,
+    ) -> None:
+        super().__init__()
+        # Do not use kwargs for now
+        assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
+
+        accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
+        accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
+        self.accelerator = accelerator
+        if accelerator.num_processes > 1:
+            self._device = torch.device(f"cuda:{accelerator.local_process_index}")
+            self.device_map = f"cuda:{accelerator.local_process_index}"
+        elif accelerator.num_processes == 1 and device_map == "auto":
+            self._device = torch.device(device)
+            self.device_map = device_map
+        else:
+            self._device = torch.device(f"cuda:{accelerator.local_process_index}")
+            self.device_map = f"cuda:{accelerator.local_process_index}"
+
+        llava_model_args = {
+            "multimodal": True,
+        }
+        if customized_config is not None:
+            llava_model_args["customized_config"] = customized_config
+        if attn_implementation is not None:
+            llava_model_args["attn_implementation"] = attn_implementation
+        if "use_flash_attention_2" in kwargs:
+            llava_model_args["use_flash_attention_2"] = kwargs["use_flash_attention_2"]
+        self.model_name = model_name
+        # model_name = model_name if model_name is not None else get_model_name_from_path(pretrained)
+        # model_name = "llava-v1.5-7b" if pretrained == "MBZUAI/PALO-7B" else get_model_name_from_path(pretrained)
+        try:
+            # Try to load the model with the multimodal argument
+            self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
+        except TypeError:
+            # for older versions of LLaVA that don't have multimodal argument
+            llava_model_args.pop("multimodal", None)
+            self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
+        self._config = self._model.config
+        self.model.eval()
+        if tie_weights:
+            self.model.tie_weights()
+
+        self.truncation = truncation
+        self.batch_size_per_gpu = int(batch_size)
+        self.conv_template = conv_template
+        self.use_cache = use_cache
+        self.truncate_context = truncate_context
+        # assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
+        if accelerator.num_processes > 1:
+            assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
+            # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
+            # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
+            # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
+            if accelerator.distributed_type == DistributedType.DEEPSPEED:
+                kwargs = {
+                    "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
+                    "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
+                }
+                AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
+                eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
+
+            if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
+                self._model = accelerator.prepare(self.model)
+            else:
+                self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
+            self.accelerator = accelerator
+            if self.accelerator.is_local_main_process:
+                eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
+            self._rank = self.accelerator.local_process_index
+            self._world_size = self.accelerator.num_processes
+        elif accelerator.num_processes == 1 and device_map == "auto":
+            eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
+            self._rank = 0
+            self._word_size = 1
+        else:
+            eval_logger.info(f"Using single device: {self._device}")
+            self.model.to(self._device)
+            self._rank = 0
+            self._world_size = 1
+                
+
+    @property
+    def config(self):
+        # return the associated transformers.AutoConfig for the given pretrained model.
+        return self._config
+
+    @property
+    def tokenizer(self):
+        return self._tokenizer
+
+    @property
+    def model(self):
+        # returns the model, unwrapping it if using Accelerate
+        if hasattr(self, "accelerator"):
+            return self.accelerator.unwrap_model(self._model)
+        else:
+            return self._model
+
+    @property
+    def eot_token_id(self):
+        # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
+        return self.tokenizer.eos_token_id
+
+    @property
+    def max_length(self):
+        return self._max_length
+
+    def pad_sequence(self, input_ids, batch_first, padding_value):
+        if self.tokenizer.padding_side == "left":
+            input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
+        input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
+        if self.tokenizer.padding_side == "left":
+            input_ids = torch.flip(input_ids, [1])
+        return input_ids
+
+    @property
+    def batch_size(self):
+        return self.batch_size_per_gpu
+
+    @property
+    def device(self):
+        return self._device
+
+    @property
+    def rank(self):
+        return self._rank
+
+    @property
+    def world_size(self):
+        return self._world_size
+
+    def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
+        """ """
+        add_special_tokens = False if add_special_tokens is None else add_special_tokens
+        encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
+        # left-truncate the encoded context to be at most `left_truncate_len` tokens long
+        if left_truncate_len:
+            encoding = encoding[-left_truncate_len:]
+        return encoding
+
+    def tok_decode(self, tokens):
+        try:
+            return self.tokenizer.decode(tokens)
+        except:
+            return self.tokenizer.decode([tokens])
+
+    def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
+        # TODO
+        res = []
+        pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
+
+        for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
+            # encode, pad, and truncate contexts for this batch
+            if type(doc_to_target) == str:
+                continuation = doc_to_target
+            else:
+                continuation = doc_to_target(self.task_dict[task][split][doc_id])
+            visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
+            visuals = self.flatten(visuals)
+            image_sizes = [[visual.size[0], visual.size[1]] for visual in visuals]
+            if visuals:
+                image = process_images(visuals, self._image_processor, self._config)
+                if type(image) is list:
+                    image = [_image.to(dtype=torch.float16, device=self.device) for _image in image]
+                else:
+                    image = image.to(dtype=torch.float16, device=self.device)
+            else:
+                image = None
+
+            prompts_input = contexts[0] if isinstance(contexts, list) else contexts
+
+            if image is not None and len(image) != 0 and DEFAULT_IMAGE_TOKEN not in prompts_input:
+                """
+                Three senarios:
+                1. No image, and there for, no image token should be added.
+                2. image token is already specified in the context, so we don't need to add it.
+                3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
+                """
+                image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visuals)
+                image_tokens = " ".join(image_tokens)
+                prompts_input = image_tokens + "\n" + (contexts[0] if isinstance(contexts, list) else contexts)
+
+            # This is much safer for llama3, as we now have some object type in it
+            if "llama_3" in self.conv_template:
+                conv = copy.deepcopy(conv_templates[self.conv_template])
+            else:
+                conv = conv_templates[self.conv_template].copy()
+            conv.append_message(conv.roles[0], prompts_input)
+            conv.append_message(conv.roles[1], None)
+            prompt = conv.get_prompt()
+            pad_token_id = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
+            contxt_id = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
+            # Add the answer of the second role
+            conv.messages[1][1] = continuation
+
+            prompt = conv.get_prompt()
+            input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
+            labels = input_ids.clone()
+            # Context part no need to calculate for loss
+            labels[0, : contxt_id.shape[1]] = -100
+            with torch.inference_mode():
+                outputs = self.model(input_ids=input_ids, labels=labels, images=image, use_cache=True, image_sizes=image_sizes)
+            loss = outputs["loss"]
+            # loss = torch.exp(loss)
+            logits = outputs["logits"]
+            greedy_tokens = logits.argmax(dim=-1)
+            cont_toks = input_ids[:, contxt_id.shape[1] :]  # [1, seq]
+            greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : input_ids.shape[1]]  # [1, seq]
+            max_equal = (greedy_tokens == cont_toks).all()
+            res.append((float(loss.item()), bool(max_equal)))
+            pbar.update(1)
+        pbar.close()
+        return res
+
+    def flatten(self, input):
+        new_list = []
+        for i in input:
+            for j in i:
+                new_list.append(j)
+        return new_list
+
+    def generate_until(self, requests: List[Instance]) -> List[str]:
+        res = []
+
+        def _collate(x):
+            # the negative sign on len(toks) sorts descending - this has a few advantages:
+            # - time estimates will always be over not underestimates, which is more useful for planning
+            # - to know the size of a batch when going through the list, you know the first one is always the batch
+            #   padded context length. this is useful to simplify the batching logic and more importantly to make
+            #   automatic adaptive batches much much easier to implement
+            # - any OOMs will happen right away rather than near the end
+            toks = self.tok_encode(x[0])
+            return -len(toks), x[0]
+
+        # we group requests by their generation_kwargs,
+        # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
+        # in the same batch.
+        re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
+        chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
+        num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
+        pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
+        for chunk in chunks:
+            contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
+            task = task[0]
+            split = split[0]
+            batched_visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]  # [B, N]
+            flattened_visuals = self.flatten(batched_visuals)  # [B*N]
+            # we assume all gen kwargs in the batch are the same
+            # this is safe to assume because the `grouper` object ensures it.
+            gen_kwargs = all_gen_kwargs[0]
+
+            # Set default values for until and max_new_tokens
+            until = [self.tok_decode(self.eot_token_id)]
+
+            # Update values from gen_kwargs if present
+            if "until" in gen_kwargs:
+                until = gen_kwargs.pop("until")
+                if isinstance(until, str):
+                    until = [until]
+                elif not isinstance(until, list):
+                    raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
+
+            if "image_aspect_ratio" in gen_kwargs.keys() and "image_aspect_ratio" not in self._config.__dict__:
+                # here we should pop it out of gen_kwargs so that it doesn't get passed to the model for next step of generation
+                self._config.image_aspect_ratio = gen_kwargs.pop("image_aspect_ratio")
+                eval_logger.info(f"Setting image aspect ratio: {self._config.image_aspect_ratio}")
+            # encode, pad, and truncate contexts for this batch
+            if flattened_visuals:
+                image_tensor = process_images(flattened_visuals, self._image_processor, self._config)
+                if type(image_tensor) is list:
+                    image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor]
+                else:
+                    image_tensor = image_tensor.to(dtype=torch.float16, device=self.device)
+            else:
+                image_tensor = None
+
+            # prompts_input = contexts[0]
+
+            question_input = []
+
+            for visual, context in zip(batched_visuals, contexts):
+                if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context:
+                    """
+                    Three senarios:
+                    1. No image, and there for, no image token should be added.
+                    2. image token is already specified in the context, so we don't need to add it.
+                    3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
+                    """
+                    image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visual) if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
+                    image_tokens = " ".join(image_tokens)
+                    question = image_tokens + "\n" + context
+                else:
+                    question = context
+                # This is much safer for llama3, as we now have some object type in it
+                if "llama_3" in self.conv_template:
+                    conv = copy.deepcopy(conv_templates[self.conv_template])
+                else:
+                    conv = conv_templates[self.conv_template].copy()
+                conv.append_message(conv.roles[0], question)
+                conv.append_message(conv.roles[1], None)
+                prompt_question = conv.get_prompt()
+                question_input.append(prompt_question)
+
+            # input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
+            # preconfigure gen_kwargs with defaults
+            gen_kwargs["image_sizes"] = [flattened_visuals[idx].size for idx in range(len(flattened_visuals))]
+            if "max_new_tokens" not in gen_kwargs:
+                gen_kwargs["max_new_tokens"] = 1024
+            if "temperature" not in gen_kwargs:
+                gen_kwargs["temperature"] = 0
+            if "top_p" not in gen_kwargs:
+                gen_kwargs["top_p"] = None
+            if "num_beams" not in gen_kwargs:
+                gen_kwargs["num_beams"] = 1
+
+            input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
+            pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
+            input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
+            attention_masks = input_ids.ne(pad_token_ids).to(self.device)
+            # These steps are not in LLaVA's original code, but are necessary for generation to work
+            # TODO: attention to this major generation step...
+            try:
+                cont = self.model.generate(
+                    input_ids,
+                    attention_mask=attention_masks,
+                    pad_token_id=pad_token_ids,
+                    images=image_tensor,
+                    image_sizes=gen_kwargs["image_sizes"],
+                    do_sample=True if gen_kwargs["temperature"] > 0 else False,
+                    temperature=gen_kwargs["temperature"],
+                    top_p=gen_kwargs["top_p"],
+                    num_beams=gen_kwargs["num_beams"],
+                    max_new_tokens=gen_kwargs["max_new_tokens"],
+                    use_cache=self.use_cache,
+                )
+                text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
+            except Exception as e:
+                raise e
+                eval_logger.error(f"Error {e} in generating")
+                cont = ""
+                text_outputs = [""]
+
+            # cont_toks_list = cont.tolist()
+            # for cont_toks, context in zip(cont_toks_list, contexts):
+            # discard context + left-padding toks if using causal decoder-only LMM
+            # if self.truncate_context:
+            #     cont_toks = cont_toks[input_ids.shape[1] :]
+            # use secondary stop seqs to cut off should-have-been-stopped content post-hoc
+            # if self.truncate_context:
+            #     for term in until:
+            #         if len(term) > 0:
+            #             # ignore '' separator,
+            #             # for seq2seq case where self.tok_decode(self.eot_token_id) = ''
+            #             text_outputs = text_outputs.split(term)[0]
+            res.extend(text_outputs)
+            self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
+            pbar.update(1)
+            # reorder this group of results back to original unsorted form
+        res = re_ords.get_original(res)
+
+        pbar.close()
+        return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+            raise NotImplementedError("TODO: Implement multi-round generation for Pangea.")
diff --git a/lmms_eval/tasks/cvqa/cvqa.yaml b/lmms_eval/tasks/cvqa/cvqa.yaml
new file mode 100644
index 000000000..e8cb76553
--- /dev/null
+++ b/lmms_eval/tasks/cvqa/cvqa.yaml
@@ -0,0 +1,31 @@
+dataset_path: afaji/cvqa
+dataset_name: default
+task: "cvqa"
+dataset_kwargs:
+  token: True
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.cvqa_doc_to_visual
+doc_to_text: !function utils.cvqa_doc_to_text
+doc_to_target: !function utils.cvqa_doc_to_target
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  do_sample: False
+metric_list:
+  - metric: cvqa_passthrough
+    aggregation: !function utils.cvqa_test_aggregation_result
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+process_results: !function utils.cvqa_process_results
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    format: default
+    translated: False 
+model_specific_generation_kwargs:
+  llava:
+    image_aspect_ratio: original
\ No newline at end of file
diff --git a/lmms_eval/tasks/cvqa/cvqa_translated.yaml b/lmms_eval/tasks/cvqa/cvqa_translated.yaml
new file mode 100644
index 000000000..c8619961f
--- /dev/null
+++ b/lmms_eval/tasks/cvqa/cvqa_translated.yaml
@@ -0,0 +1,31 @@
+dataset_path: afaji/cvqa
+dataset_name: default
+task: "cvqa_translated"
+dataset_kwargs:
+  token: True
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.cvqa_doc_to_visual
+doc_to_text: !function utils.cvqa_doc_to_text
+doc_to_target: !function utils.cvqa_doc_to_target
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  do_sample: False
+metric_list:
+  - metric: cvqa_passthrough
+    aggregation: !function utils.cvqa_test_aggregation_result
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+process_results: !function utils.cvqa_process_results
+metadata:
+  - version: 0.0
+
+model_specific_prompt_kwargs:
+  default:
+    format: default
+    translated: True
+model_specific_generation_kwargs:
+  llava:
+    image_aspect_ratio: original
\ No newline at end of file
diff --git a/lmms_eval/tasks/cvqa/utils.py b/lmms_eval/tasks/cvqa/utils.py
new file mode 100644
index 000000000..91228062c
--- /dev/null
+++ b/lmms_eval/tasks/cvqa/utils.py
@@ -0,0 +1,90 @@
+import re
+import random
+from difflib import SequenceMatcher
+import os
+
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+
+def cvqa_doc_to_text(doc, model_specific_prompt_kwargs):
+    if model_specific_prompt_kwargs["translated"] is True:
+        print("Using translated prompt")
+        question, choices = doc["Translated Question"], doc["Translated Options"]
+        doc["Question"] = doc["Translated Question"]
+        doc["Options"] = doc["Translated Options"]
+    else:
+        question, choices = doc["Question"], doc["Options"]
+    len_choices = len(choices)
+    options = [chr(ord("A") + i) for i in range(len_choices)]
+    choices_str = "\n".join([f"{option}. {choice}" for option, choice in zip(options, choices)])
+    return f"Question: {question} \n\n Options: {choices_str} \n\n Answer with the option's letter from the given choices directly."
+
+
+def cvqa_doc_to_visual(doc):
+    if doc["image"] is None:
+        return []
+    return [doc["image"].convert("RGB")]
+
+
+def cvqa_doc_to_target(doc):
+    len_choices = len(doc["Options"])
+    options = [chr(ord("A") + i) for i in range(len_choices)]
+    return options[doc["Label"]]
+
+
+def similar(a, b):
+    return SequenceMatcher(None, a, b).ratio()
+
+def parse_multi_choice_response(response, options):
+    response = response.strip()
+    
+    # Original letter-matching logic
+    match = re.search(r'\(?([A-D])[).:\s]', response, re.IGNORECASE)
+    if match:
+        return match.group(1).upper()
+    
+    # If no match found, fallback to searching for any A, B, C, or D in the response
+    match = re.search(r'[ABCD]', response, re.IGNORECASE)
+    if match:
+        return match.group(0).upper()
+
+    # If no letter found, match full content
+    best_match = None
+    best_match_ratio = 0
+    for i, option in enumerate(options):
+        option_content = re.sub(r'^[A-D]\.\s*', '', option).strip()
+        similarity = similar(response, option_content)
+        if similarity > best_match_ratio:
+            best_match = chr(65 + i)  # 'A', 'B', 'C', or 'D'
+            best_match_ratio = similarity
+
+    # If we found a good match (you can adjust the threshold)
+    if best_match_ratio > 0.7:
+        return best_match
+
+    # If all else fails, return a random choice
+    return random.choice(['A', 'B', 'C', 'D'])
+
+def cvqa_process_results(doc, results):
+    # I know this is weird, but it's how llava parse it.
+    target = cvqa_doc_to_target(doc)
+    pred = parse_multi_choice_response(results[0],doc['Options'])
+    pred_numerical = {'A':0, 'B':1, 'C':2, 'D':3}[pred]
+    results_dict = {"cvqa_passthrough": {"id": doc["ID"], "pred": pred_numerical, "target": target}}
+    return results_dict
+
+    # with open (output_dir + "/cvqa_predictions.csv", "a") as f:
+    #     f.write(f"{doc['ID']},{pred_numerical}\n")
+    # if pred == target:
+    #     return {"exact_match": 1.0}
+    # # pattern: ^[A-Z]\. .*
+    # if len(pred) >= 2 and pred[0].isupper() and pred[1] == ".": 
+    #     result = 1.0 if pred[0] == target else 0.0
+    #     return {"exact_match": result}
+    # return {"exact_match": 0.0}
+
+def cvqa_test_aggregation_result(results, args):
+    path = generate_submission_file("cvqa_predictions.csv", args)
+    for result in results:
+        with open(path, "a") as f:
+            f.write(f"{result['id']},{result['pred']}\n")
+
diff --git a/lmms_eval/tasks/m3exam/m3exam.yaml b/lmms_eval/tasks/m3exam/m3exam.yaml
new file mode 100644
index 000000000..26481ea5f
--- /dev/null
+++ b/lmms_eval/tasks/m3exam/m3exam.yaml
@@ -0,0 +1,9 @@
+group: m3exam
+task:
+- m3exam_chinese
+- m3exam_english
+- m3exam_afrikaans
+- m3exam_italian
+- m3exam_portuguese
+- m3exam_thai
+- m3exam_vietnamese
diff --git a/lmms_eval/tasks/m3exam/m3exam_afrikaans.yaml b/lmms_eval/tasks/m3exam/m3exam_afrikaans.yaml
new file mode 100644
index 000000000..151cbc8db
--- /dev/null
+++ b/lmms_eval/tasks/m3exam/m3exam_afrikaans.yaml
@@ -0,0 +1,20 @@
+dataset_path: neulab/PangeaBench-m3exam
+task: "m3exam_afrikaans"
+test_split: afrikaans
+output_type: generate_until
+doc_to_visual: !function utils.m3exam_doc_to_visual
+doc_to_text: !function utils.m3exam_doc_to_text
+doc_to_target: "answer_text"
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.m3exam_process_results
+metric_list:
+  - metric: m3exam
+    aggregation: !function utils.m3exam_aggregate_results
+    higher_is_better: true
+metadata:
+  - version: 0.0
diff --git a/lmms_eval/tasks/m3exam/m3exam_chinese.yaml b/lmms_eval/tasks/m3exam/m3exam_chinese.yaml
new file mode 100644
index 000000000..e760ae632
--- /dev/null
+++ b/lmms_eval/tasks/m3exam/m3exam_chinese.yaml
@@ -0,0 +1,20 @@
+dataset_path: neulab/PangeaBench-m3exam
+task: "m3exam_chinese"
+test_split: chinese
+output_type: generate_until
+doc_to_visual: !function utils.m3exam_doc_to_visual
+doc_to_text: !function utils.m3exam_doc_to_text
+doc_to_target: "answer_text"
+generation_kwargs:
+  max_new_tokens: 32
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.m3exam_process_results
+metric_list:
+  - metric: m3exam
+    aggregation: !function utils.m3exam_aggregate_results
+    higher_is_better: true
+metadata:
+  - version: 0.0
diff --git a/lmms_eval/tasks/m3exam/m3exam_english.yaml b/lmms_eval/tasks/m3exam/m3exam_english.yaml
new file mode 100644
index 000000000..533f56b11
--- /dev/null
+++ b/lmms_eval/tasks/m3exam/m3exam_english.yaml
@@ -0,0 +1,20 @@
+dataset_path: neulab/PangeaBench-m3exam
+task: "m3exam_english"
+test_split: english
+output_type: generate_until
+doc_to_visual: !function utils.m3exam_doc_to_visual
+doc_to_text: !function utils.m3exam_doc_to_text
+doc_to_target: "answer_text"
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.m3exam_process_results
+metric_list:
+  - metric: m3exam
+    aggregation: !function utils.m3exam_aggregate_results
+    higher_is_better: true
+metadata:
+  - version: 0.0
diff --git a/lmms_eval/tasks/m3exam/m3exam_italian.yaml b/lmms_eval/tasks/m3exam/m3exam_italian.yaml
new file mode 100644
index 000000000..5dc214598
--- /dev/null
+++ b/lmms_eval/tasks/m3exam/m3exam_italian.yaml
@@ -0,0 +1,20 @@
+dataset_path: neulab/PangeaBench-m3exam
+task: "m3exam_italian"
+test_split: italian
+output_type: generate_until
+doc_to_visual: !function utils.m3exam_doc_to_visual
+doc_to_text: !function utils.m3exam_doc_to_text
+doc_to_target: "answer_text"
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.m3exam_process_results
+metric_list:
+  - metric: m3exam
+    aggregation: !function utils.m3exam_aggregate_results
+    higher_is_better: true
+metadata:
+  - version: 0.0
diff --git a/lmms_eval/tasks/m3exam/m3exam_portuguese.yaml b/lmms_eval/tasks/m3exam/m3exam_portuguese.yaml
new file mode 100644
index 000000000..160e44bd2
--- /dev/null
+++ b/lmms_eval/tasks/m3exam/m3exam_portuguese.yaml
@@ -0,0 +1,20 @@
+dataset_path: neulab/PangeaBench-m3exam
+task: "m3exam_portuguese"
+test_split: portuguese
+output_type: generate_until
+doc_to_visual: !function utils.m3exam_doc_to_visual
+doc_to_text: !function utils.m3exam_doc_to_text
+doc_to_target: "answer_text"
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.m3exam_process_results
+metric_list:
+  - metric: m3exam
+    aggregation: !function utils.m3exam_aggregate_results
+    higher_is_better: true
+metadata:
+  - version: 0.0
diff --git a/lmms_eval/tasks/m3exam/m3exam_thai.yaml b/lmms_eval/tasks/m3exam/m3exam_thai.yaml
new file mode 100644
index 000000000..cf0320069
--- /dev/null
+++ b/lmms_eval/tasks/m3exam/m3exam_thai.yaml
@@ -0,0 +1,20 @@
+dataset_path: neulab/PangeaBench-m3exam
+task: "m3exam_thai"
+test_split: thai
+output_type: generate_until
+doc_to_visual: !function utils.m3exam_doc_to_visual
+doc_to_text: !function utils.m3exam_doc_to_text
+doc_to_target: "answer_text"
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.m3exam_process_results
+metric_list:
+  - metric: m3exam
+    aggregation: !function utils.m3exam_aggregate_results
+    higher_is_better: true
+metadata:
+  - version: 0.0
diff --git a/lmms_eval/tasks/m3exam/m3exam_vietnamese.yaml b/lmms_eval/tasks/m3exam/m3exam_vietnamese.yaml
new file mode 100644
index 000000000..8ddbc6f33
--- /dev/null
+++ b/lmms_eval/tasks/m3exam/m3exam_vietnamese.yaml
@@ -0,0 +1,20 @@
+dataset_path: neulab/PangeaBench-m3exam
+task: "m3exam_vietnamese"
+test_split: vietnamese
+output_type: generate_until
+doc_to_visual: !function utils.m3exam_doc_to_visual
+doc_to_text: !function utils.m3exam_doc_to_text
+doc_to_target: "answer_text"
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.m3exam_process_results
+metric_list:
+  - metric: m3exam
+    aggregation: !function utils.m3exam_aggregate_results
+    higher_is_better: true
+metadata:
+  - version: 0.0
diff --git a/lmms_eval/tasks/m3exam/utils.py b/lmms_eval/tasks/m3exam/utils.py
new file mode 100644
index 000000000..848b3ce22
--- /dev/null
+++ b/lmms_eval/tasks/m3exam/utils.py
@@ -0,0 +1,242 @@
+import re
+import logging
+import random
+from difflib import SequenceMatcher
+from PIL import Image as PILImage
+from PIL import ImageDraw, ImageFont
+import io
+import base64
+lmms_logger = logging.getLogger("lmms-eval")
+
+LANG_CONFIG = {
+    'english': "Answer:",
+    'chinese': '答案：',
+    'vietnamese': 'Câu trả lời:',
+    'thai': 'คำตอบ:',
+    'italian': 'La risposta:',
+    'afrikaans': 'Antwoord:',
+    'portuguese': 'Responder:'
+}
+    
+def construct_prompt(doc):
+    lang = doc["language"]
+    subject2target = {
+        'english': {'language': 'English', 'math': "Math", 'social-science': "Social Science", 'natural-science': 'Natural Science'},
+        # 'english4all': {'language': 'Language', 'math': "Math", 'social-science': "Social Science", 'natural-science': 'Natural Science'},
+        'chinese':  {'language': '语文', 'math': "数学", 'social-science': "社会科学", 'natural-science': '自然科学'},
+        'thai': {'language': 'ภาษาไทย', 'math': 'คณิตศาสตร์', 'social-science': 'สังคมศึกษา', 'natural-science': 'วิทยาศาสตร์'},
+        'vietnamese': {'language': 'Tiếng Việt', 'math': "Toán", 'social-science': "Khoa học xã hội", 'natural-science': 'Khoa học tự nhiên'},
+        'italian': {'language': 'Italiano', 'math': "Matematica", 'social-science': "Scienze sociali", 'natural-science': 'Scienze naturali'},
+        'afrikaans': {'language': 'Afrikaans Huistaal', 'math': "Wiskunde", 'social-science': "Sosiale Wetenskappe", 'natural-science': 'Natuurwetenskap'},
+        'portuguese': {'language': 'Linguagens', 'math': 'Matemática', 'social-science': 'Ciências Humanas', 'natural-science': 'Ciências da Natureza'},
+    }
+    subject = subject2target[lang][doc['subject_category']]
+
+    hint_templates = {
+        'english': f"The following is a multiple choice question about {subject}. Please only respond with the letter (A, B, C, or D) corresponding to the correct answer, without any additional explanation.",
+        'chinese': f"以下是关于{subject}的单项选择题。 请仅给出正确选项对应的选项序号（A, B, C, 或 D） 而非其他细节。",
+        'thai': f"ต่อไปนี้เป็นคำถามแบบปรนัย วิชา{subject} โปรดตอบเพียงตัวอักษร (A, B, C หรือ D) ที่ตรงกับคำตอบที่ถูกต้อง โดยไม่ต้องมีคำอธิบายเพิ่มเติม",
+        'vietnamese': f"Sau đây là câu hỏi trắc nghiệm về {subject}. Vui lòng chỉ trả lời bằng chữ cái (A, B, C hoặc D) tương ứng với câu trả lời đúng, không cần giải thích thêm.",
+        'italian': f"La seguente è una domanda a scelta multipla su {subject}. Si prega di rispondere solo con la lettera (A, B, C o D) corrispondente alla risposta corretta, senza alcuna spiegazione aggiuntiva.",
+        'afrikaans': f"Die volgende is 'n meervoudige keuse vraag oor {subject}. Antwoord asseblief slegs met die letter (A, B, C of D) wat ooreenstem met die korrekte antwoord, sonder enige bykomende verduideliking.",
+        'portuguese': f"A seguir está uma questão de múltipla escolha sobre {subject}. Por favor, responda apenas com a letra (A, B, C ou D) correspondente à resposta correta, sem qualquer explicação adicional."
+    }
+
+    hint = hint_templates.get(lang, "")
+    prompt = f"{hint}\n\n<prompt_placeholder>"
+    return prompt
+
+def similar(a, b):
+    return SequenceMatcher(None, a, b).ratio()
+
+def parse_multi_choice_response(response, options):
+    response = response.strip()
+    
+    # Original letter-matching logic
+    match = re.search(r'\(?([A-D])[).:\s]', response, re.IGNORECASE)
+    if match:
+        return match.group(1).upper()
+    
+    # If no match found, fallback to searching for any A, B, C, or D in the response
+    match = re.search(r'[ABCD]', response, re.IGNORECASE)
+    if match:
+        return match.group(0).upper()
+
+    # If no letter found, match full content
+    best_match = None
+    best_match_ratio = 0
+    for i, option in enumerate(options):
+        option_content = re.sub(r'^[A-D]\.\s*', '', option).strip()
+        similarity = similar(response, option_content)
+        if similarity > best_match_ratio:
+            best_match = chr(65 + i)  # 'A', 'B', 'C', or 'D'
+            best_match_ratio = similarity
+
+    # If we found a good match (you can adjust the threshold)
+    if best_match_ratio > 0.7:
+        return best_match
+
+    # If all else fails, return a random choice
+    return random.choice(['A', 'B', 'C', 'D'])
+
+def m3exam_process_results(doc, results):
+    pred = results[0]
+    parsed_pred = parse_multi_choice_response(pred,doc['options'])
+    standardized_answer = standardize_answer(doc["answer_text"], doc['options'])
+
+    return {
+        "m3exam": {
+            "language": doc["language"],
+            "origin_response": pred,
+            "answer_text": parsed_pred,
+            "origin_answer": doc["answer_text"],
+            "standardized_answer": standardized_answer
+        }
+    }
+
+def m3exam_aggregate_results(results):
+    total, match = 0, 0
+    for question in results:
+        total += 1
+        if question["answer_text"] == question["standardized_answer"]:
+            match += 1
+    
+    accuracy = match / total if total > 0 else 0
+    print(f"==========================")
+    print(f"========Final Score=======")
+    print(f"Total questions: {total}")
+    print(f"Correct answers: {match}")
+    print(f"Accuracy: {accuracy:.2%}")
+    print(f"==========================")
+    
+    return accuracy
+
+
+def replace_images_tokens(input_string, image_ids):
+    if len(image_ids) == 0:
+        return input_string
+    else:
+        for id in image_ids:
+            image_str = f"(image)[{id}]"
+            query_text = "<image>"
+            if image_str in input_string:
+                input_string = input_string.replace(image_str, query_text)
+    return input_string
+
+def standardize_options(options):
+    standardized = []
+    for i, option in enumerate(options):
+        # Remove any existing option identifier
+        cleaned = re.sub(r'^[\(（]?[A-Da-d\d][\)）\.\s]+\s*', '', option).strip()
+        # Add standardized identifier
+        standardized.append(f"({chr(65+i)}) {cleaned}")
+    return standardized
+
+
+def standardize_answer(answer, options):
+    # If answer is already A, B, C, or D, return it
+    if answer in 'ABCDEFGH':
+        return answer
+    
+    # If answer is 1, 2, 3, or 4, convert to A, B, C, or D
+    if answer in '12345678':
+        return chr(64 + int(answer))
+    
+    # If answer is (1), (2), (3), or (4), convert to A, B, C, or D
+    match = re.match(r'\(?(\d)\)?', answer)
+    if match:
+        return chr(64 + int(match.group(1)))
+    
+    # If we can't determine the answer, return the original
+    return answer
+
+def get_text_dimensions(text, font):
+    # This function handles both newer and older Pillow versions
+    if hasattr(font, "getbbox"):
+        bbox = font.getbbox(text)
+        return bbox[2] - bbox[0], bbox[3] - bbox[1]
+    else:
+        return font.getsize(text)
+
+def combine_images_with_labels(images, labels):
+    """
+    Combine multiple images into a single image, adding labels with dynamically sized font.
+    """
+    max_width = max(img.width for img in images)
+    total_height = sum(img.height for img in images)
+    padding = 20  # Increased padding for larger labels
+
+    combined_img = PILImage.new('RGB', (max_width, total_height + padding * (len(images) - 1)), color='white')
+    draw = ImageDraw.Draw(combined_img)
+    
+    # Calculate dynamic font size
+    base_font_size = int(min(max_width, total_height) * 0.03)  # 3% of the smaller dimension
+    font_size = max(24, min(base_font_size, 72))  # Minimum 24, maximum 72
+    
+    try:
+        font = ImageFont.truetype("OpenSans-Regular.ttf", font_size)
+    except IOError:
+        font = ImageFont.load_default()
+
+    y_offset = 0
+    for i, img in enumerate(images):
+        combined_img.paste(img, (0, y_offset))
+        label = labels[i]
+        
+        # Calculate text position
+        text_width, text_height = get_text_dimensions(label, font)
+        text_x = 0  # 10 pixels from the left edge
+        text_y = y_offset - 10  # 10 pixels from the top of each image
+        
+        # Add a semi-transparent background for the text for better readability
+        text_bg = PILImage.new('RGBA', (text_width + 20, text_height + 20), (255, 255, 255, 128))
+        combined_img.paste(text_bg, (text_x - 10, text_y - 10), text_bg)
+        
+        # Draw the text
+        draw.text((text_x, text_y), label, fill="black", font=font)
+        
+        y_offset += img.height + padding
+
+    return combined_img
+
+def generate_white_image(width=300, height=200):
+    """Generate a white image of the specified size."""
+    return PILImage.new('RGB', (width, height), color='white')
+
+def m3exam_doc_to_visual(doc):
+    visual = []
+    for i in range(10):
+        if doc['image_' + str(i)] == "None":
+            continue
+        else:
+            image_data = base64.b64decode(doc['image_' + str(i)])
+            image = PILImage.open(io.BytesIO(image_data))
+    
+            image_rgb = image.convert('RGB')
+
+            visual.append(image_rgb)
+
+    return visual
+    
+def m3exam_doc_to_text(doc):
+    # Standardize options
+    doc['options'] = standardize_options(doc['options'])
+    lang = doc["language"]
+
+    # Process question text
+    question_text = re.sub(r'\(image\)\[[^\]]+\]', '<image>', doc['question_text'], count=1)
+    question_text = re.sub(r'\(image\)\[[^\]]+\]', '', question_text)
+    
+    # Prepare options text
+    if len(re.findall(r'\(image\)\[[^\]]+\]', ' '.join(doc['options']))) > 0:
+        # If options contain images, don't include option text
+        options_text = ''
+    else:
+        options_text = '\n'.join(doc['options'])
+    
+    # Construct prompt
+    prompt = construct_prompt(doc)
+    background = '\n' + '\n'.join(doc['background_description']) if doc['background_description'] else ''
+    prompt = prompt.replace("<prompt_placeholder>", f"{background}\n{question_text}\n{options_text}\n{LANG_CONFIG[lang]}")
+
+    return prompt
\ No newline at end of file
diff --git a/lmms_eval/tasks/marvl/marvl.yaml b/lmms_eval/tasks/marvl/marvl.yaml
new file mode 100755
index 000000000..05f669d73
--- /dev/null
+++ b/lmms_eval/tasks/marvl/marvl.yaml
@@ -0,0 +1,8 @@
+group: marvl
+task:
+  - marvl_id
+  - marvl_sw
+  - marvl_ta
+  - marvl_tr
+  - marvl_zh
+  - nlvr2
diff --git a/lmms_eval/tasks/marvl/marvl_id.yaml b/lmms_eval/tasks/marvl/marvl_id.yaml
new file mode 100755
index 000000000..68d8ba0d7
--- /dev/null
+++ b/lmms_eval/tasks/marvl/marvl_id.yaml
@@ -0,0 +1,30 @@
+dataset_path: neulab/PangeaBench-marvl
+dataset_name: default
+dataset_kwargs:
+  token: True
+task: "marvl_id"
+test_split: id
+output_type: generate_until
+doc_to_visual: !function utils.marvl_doc_to_visual
+doc_to_text: !function utils.marvl_doc_to_text
+doc_to_target: !function utils.marvl_doc_to_target
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+process_results: !function utils.marvl_process_result
+metadata:
+  - version: 0.0
+  
+lmms_eval_specific_kwargs:
+  default:
+    hypothesis_prompt: "Hypothesis: "
+    is_our_model: True
diff --git a/lmms_eval/tasks/marvl/marvl_sw.yaml b/lmms_eval/tasks/marvl/marvl_sw.yaml
new file mode 100755
index 000000000..45bbb3a3a
--- /dev/null
+++ b/lmms_eval/tasks/marvl/marvl_sw.yaml
@@ -0,0 +1,31 @@
+dataset_path: neulab/PangeaBench-marvl
+dataset_name: default
+dataset_kwargs:
+  token: True
+task: "marvl_sw"
+test_split: sw
+output_type: generate_until
+doc_to_visual: !function utils.marvl_doc_to_visual
+doc_to_text: !function utils.marvl_doc_to_text
+doc_to_target: !function utils.marvl_doc_to_target
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+process_results: !function utils.marvl_process_result
+metadata:
+  - version: 0.0
+  
+lmms_eval_specific_kwargs:
+  default:
+    hypothesis_prompt: "Hypothesis: "
+    is_our_model: True
+    
diff --git a/lmms_eval/tasks/marvl/marvl_ta.yaml b/lmms_eval/tasks/marvl/marvl_ta.yaml
new file mode 100755
index 000000000..40ddfdf79
--- /dev/null
+++ b/lmms_eval/tasks/marvl/marvl_ta.yaml
@@ -0,0 +1,31 @@
+dataset_path: neulab/PangeaBench-marvl
+dataset_name: default
+dataset_kwargs:
+  token: True
+task: "marvl_ta"
+test_split: ta
+output_type: generate_until
+doc_to_visual: !function utils.marvl_doc_to_visual
+doc_to_text: !function utils.marvl_doc_to_text
+doc_to_target: !function utils.marvl_doc_to_target
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+process_results: !function utils.marvl_process_result
+metadata:
+  - version: 0.0
+  
+lmms_eval_specific_kwargs:
+  default:
+    hypothesis_prompt: "Hypothesis: "
+    is_our_model: True
+    
diff --git a/lmms_eval/tasks/marvl/marvl_tr.yaml b/lmms_eval/tasks/marvl/marvl_tr.yaml
new file mode 100755
index 000000000..4cdee0aed
--- /dev/null
+++ b/lmms_eval/tasks/marvl/marvl_tr.yaml
@@ -0,0 +1,31 @@
+dataset_path: neulab/PangeaBench-marvl
+dataset_name: default
+dataset_kwargs:
+  token: True
+task: "marvl_tr"
+test_split: tr
+output_type: generate_until
+doc_to_visual: !function utils.marvl_doc_to_visual
+doc_to_text: !function utils.marvl_doc_to_text
+doc_to_target: !function utils.marvl_doc_to_target
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+process_results: !function utils.marvl_process_result
+metadata:
+  - version: 0.0
+  
+lmms_eval_specific_kwargs:
+  default:
+    hypothesis_prompt: "Hypothesis: "
+    is_our_model: True
+    
diff --git a/lmms_eval/tasks/marvl/marvl_zh.yaml b/lmms_eval/tasks/marvl/marvl_zh.yaml
new file mode 100755
index 000000000..4e3253bfc
--- /dev/null
+++ b/lmms_eval/tasks/marvl/marvl_zh.yaml
@@ -0,0 +1,31 @@
+dataset_path: neulab/PangeaBench-marvl
+dataset_name: default
+dataset_kwargs:
+  token: True
+task: "marvl_zh"
+test_split: zh
+output_type: generate_until
+doc_to_visual: !function utils.marvl_doc_to_visual
+doc_to_text: !function utils.marvl_doc_to_text
+doc_to_target: !function utils.marvl_doc_to_target
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+process_results: !function utils.marvl_process_result
+metadata:
+  - version: 0.0
+  
+lmms_eval_specific_kwargs:
+  default:
+    hypothesis_prompt: "Hypothesis: "
+    is_our_model: True
+    
diff --git a/lmms_eval/tasks/marvl/nlvr2.yaml b/lmms_eval/tasks/marvl/nlvr2.yaml
new file mode 100755
index 000000000..fabe48a95
--- /dev/null
+++ b/lmms_eval/tasks/marvl/nlvr2.yaml
@@ -0,0 +1,30 @@
+dataset_path: neulab/PangeaBench-marvl
+dataset_name: default
+dataset_kwargs:
+  token: True
+task: "nlvr2"
+test_split: en
+output_type: generate_until
+doc_to_visual: !function utils.marvl_doc_to_visual
+doc_to_text: !function utils.marvl_doc_to_text
+doc_to_target: !function utils.marvl_doc_to_target
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+process_results: !function utils.marvl_process_result
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    hypothesis_prompt: "Hypothesis: "   
+    is_our_model: True
diff --git a/lmms_eval/tasks/marvl/utils.py b/lmms_eval/tasks/marvl/utils.py
new file mode 100755
index 000000000..5d1e06c19
--- /dev/null
+++ b/lmms_eval/tasks/marvl/utils.py
@@ -0,0 +1,25 @@
+
+def marvl_doc_to_visual(doc):
+    return [doc["image"].convert("RGB")]
+
+def marvl_doc_to_text(doc, model_specific_prompt_kwargs):
+    conversations = doc['conversations']
+    query = conversations[0]['value'].replace('<image>\n', '').strip() + '. Answer with the option directly.'
+    return query
+    
+def marvl_doc_to_target(doc):
+    conversations = doc['conversations']
+    answer = str(conversations[1])
+    if 'true' in answer: return 'true'
+    elif 'false' in answer: return 'false'
+    else: raise Exception(f"get target failed for id {doc['id']} - conversations: {conversations}")
+
+def marvl_process_result(doc, results):
+    target = marvl_doc_to_target(doc)
+    pred = results[0]
+    pred = pred.strip().lower()
+    if 'yes' in pred: pred = 'true'
+    elif 'no' in pred: pred = 'false'
+    if target.strip().lower() in pred:
+        return {"exact_match": 1.0}
+    return {"exact_match": 0.0}
diff --git a/lmms_eval/tasks/maxm/evaluation.py b/lmms_eval/tasks/maxm/evaluation.py
new file mode 100644
index 000000000..079e4e4ce
--- /dev/null
+++ b/lmms_eval/tasks/maxm/evaluation.py
@@ -0,0 +1,1382 @@
+import re
+from typing import Dict, List, Literal, Set
+
+import evaluate
+import numpy as np
+import pandas as pd
+from pycocoevalcap.cider.cider import Cider
+from pycocoevalcap.eval import COCOEvalCap
+from pycocoevalcap.spice.spice import Spice
+from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer
+
+
+class MyCOCOEvalCap(COCOEvalCap):
+    def __init__(
+        self,
+        coco,
+        cocoRes,
+        metrics: Set[str] = {
+            "CIDEr",
+            "SPICE",
+        },
+    ):
+        super().__init__(coco, cocoRes)
+        self.metrics = metrics
+
+    def evaluate(self):
+        imgIds = self.params["image_id"]
+        # imgIds = self.coco.getImgIds()
+        gts = {}
+        res = {}
+        for imgId in imgIds:
+            gts[imgId] = self.coco.imgToAnns[imgId]
+            res[imgId] = self.cocoRes.imgToAnns[imgId]
+
+        # =================================================
+        # Set up tokenizers
+        # =================================================
+        print("tokenization...")
+        tokenizer = PTBTokenizer()
+        gts = tokenizer.tokenize(gts)
+        res = tokenizer.tokenize(res)
+
+        # =================================================
+        # Set up scorers
+        # =================================================
+        print("setting up scorers...")
+        scorers = []
+        for metric in self.metrics:
+            if metric == "CIDEr":
+                scorers.append((Cider(), "CIDEr"))
+            elif metric == "SPICE":
+                scorers.append((Spice(), "SPICE"))
+
+        # =================================================
+        # Compute scores
+        # =================================================
+        for scorer, method in scorers:
+            print("computing %s score..." % (scorer.method()))
+            score, scores = scorer.compute_score(gts, res)
+            if isinstance(method, list):
+                for sc, scs, m in zip(score, scores, method):
+                    self.setEval(sc, m)
+                    self.setImgToEvalImgs(scs, gts.keys(), m)
+                    print("%s: %0.3f" % (m, sc))
+            else:
+                self.setEval(score, method)
+                self.setImgToEvalImgs(scores, gts.keys(), method)
+                print("%s: %0.3f" % (method, score))
+        self.setEvalImgs()
+
+
+def generated_caption_evaluation(
+    caption_preds: List[str],
+    caption_golds: List[str] | List[List[str]],
+    lang_id: str,
+    metrics: Set[str] | None = None,
+    use_gpu: bool = True,
+) -> Dict[str, float | str]:
+    if metrics is None:
+        metrics = {"bleu", "meteor", "rouge", "bertscore", "chrf"}
+        # "CIDEr", "SPICE",
+
+    if isinstance(caption_golds[0], list):
+        # When we have multiple gold captions, ChrF, as implemented by sacrebleu,
+        # requires the same number of references for each prediction
+        max_len = max(len(x) for x in caption_golds)
+        for i in range(len(caption_golds)):
+            caption_golds[i] = caption_golds[i] + [""] * (
+                max_len - len(caption_golds[i])
+            )
+
+    results = dict()
+    if "bleu" in metrics:
+        bleu = evaluate.load("bleu")
+        for bleu_n in [1, 2, 3, 4]:
+            res = bleu.compute(
+                predictions=caption_preds,
+                references=caption_golds,
+                max_order=bleu_n,
+            )
+            results[f"bleu_{bleu_n}"] = res["bleu"]
+    if "rouge" in metrics:
+        rouge = evaluate.load("rouge")
+        res = rouge.compute(
+            predictions=caption_preds,
+            references=caption_golds,
+        )
+        for rouge_method, rouge_res in res.items():
+            results[rouge_method] = rouge_res
+    if "meteor" in metrics:
+        meteor = evaluate.load("meteor")
+        res = meteor.compute(
+            predictions=caption_preds,
+            references=caption_golds,
+        )
+        results["meteor"] = res["meteor"]
+    if "chrf" in metrics:
+        chrf = evaluate.load("chrf")
+
+        for m, wo in {"": 0, "+": 1, "++": 2}.items():
+            res = chrf.compute(
+                predictions=caption_preds,
+                references=caption_golds,
+                word_order=wo,
+            )
+            results[f"chrF{m}"] = res["score"]
+    if "bertscore" in metrics:
+        bertscore = evaluate.load("bertscore", device="cuda:0" if use_gpu else "cpu")
+        if lang_id == "fil":
+            lang_id = "tl"
+        elif lang_id == "quz":
+            lang_id = "qu"
+        res = bertscore.compute(
+            predictions=caption_preds,
+            references=caption_golds,
+            lang=lang_id,
+            device="cuda:0" if use_gpu else "cpu",
+        )
+        for r in ["precision", "recall", "f1"]:
+            results[f"avg_bertscore_{r}"] = np.mean(res[r])
+            results[f"std_bertscore_{r}"] = np.std(res[r])
+    if "cider" in metrics or "spice" in metrics:
+        raise NotImplementedError(
+            "CIDEr and SPICE are not supported by the current implementation"
+        )
+        # normalize and tokenize captions
+        # if lang_id in {"zh", "ja", "th"}:
+        #     if lang_id == "zh":
+        #         from spacy.lang.zh import Chinese
+
+        #         tokenizer = Chinese()
+        #     elif lang_id == "ja":
+        #         from spacy.lang.ja import Japanese
+
+        #         tokenizer = Japanese()
+        #     elif lang_id == "th":
+        #         from spacy.lang.th import Thai
+
+        #         tokenizer = Thai()
+        #     else:
+        #         raise ValueError(
+        #             "This is not possible but I want to silence the type checker"
+        #         )
+
+        #     caption_preds = [
+        #         " ".join([word.text for word in tokenizer(caption)])
+        #         for caption in caption_preds
+        #     ]
+        # caption_preds = [unicodedata.normalize("NFC", cap) for cap in caption_preds]
+
+        # for i in range(5):
+        #     logging.info(
+        #         f"img id: {image_ids[i]} -- pred: {caption_preds[i]} --- gold: {caption_golds[i]}"
+        #     )
+
+        # # create a temp file to save predictions in COCO format
+        # predictions_data = dict(zip(image_ids, caption_preds))  # remove duplicate keys
+        # predictions_data = [
+        #     {"image_id": k, "caption": v} for k, v in predictions_data.items()
+        # ]
+        # with tempfile.NamedTemporaryFile(
+        #     suffix=".json", delete=False, mode="w"
+        # ) as temp_file:
+        #     predictions_fn = temp_file.name
+        #     json.dump(predictions_data, temp_file)
+
+        # # For template annotation files, we take encode in text_label which file to load (not clean but works)
+        # if "{" in annotation_file:
+        #     annotation_file = annotation_file.format(lang_id)
+        # coco = COCO(annotation_file)
+        # coco_result = coco.loadRes(predictions_fn)
+
+        # # create coco_eval object by taking coco and coco_result
+        # coco_eval = MyCOCOEvalCap(coco, coco_result, metrics)
+        # coco_eval.params["image_id"] = coco_result.getImgIds()
+
+        # # evaluate results
+        # # SPICE will take a few minutes the first time, but speeds up due to caching
+        # coco_eval.evaluate()
+        # for metric, score in coco_eval.eval.items():
+        #     results[metric] = score
+
+        # # delete temp file
+        # os.unlink(predictions_fn)
+    return results
+
+
+def generated_label_classification_evaluation(
+    gold_labels: List[str] | List[List[str]],
+    pred_labels: List[str],
+    vqa_post_process: bool = True,
+    bool_to_yes_no: bool = True,
+    entailment_to_yes_no_maybe: bool = False,
+    remove_trailing_period: bool = True,
+) -> Dict[str, float]:
+    single_answers = isinstance(gold_labels[0], str)
+    print(f"Evaluation with {single_answers=}")
+
+    df = pd.DataFrame(
+        {
+            "gold": gold_labels,
+            "pred": pred_labels,
+        }
+    )
+    if single_answers:
+        df["gold"] = df["gold"].apply(lambda x: str(x).strip())
+        if remove_trailing_period:
+            df["pred"] = df["pred"].apply(lambda x: str(x).rstrip("."))
+    else:
+        df["gold"] = df["gold"].apply(lambda x: [str(ans).strip() for ans in x])
+        if remove_trailing_period:
+            df["pred"] = df["pred"].apply(lambda x: [str(ans).rstrip(".") for ans in x])
+
+    if entailment_to_yes_no_maybe:
+
+        def _entailment_to_yes_no_maybe(x) -> Literal["yes", "no", "maybe"]:
+            if x.lower() in ["yes", "no", "maybe"]:
+                return x.lower()
+            elif x.lower() in ["true", "false"]:
+                return "yes" if x.lower() == "true" else "no"
+            elif x.lower() in ["1", "0"]:
+                return "yes" if x.lower() == "1" else "no"
+            elif x.lower() in ["entailment", "contradiction", "neutral"]:
+                if x.lower() == "entailment":
+                    return "yes"
+                elif x.lower() == "contradiction":
+                    return "no"
+                else:
+                    return "maybe"
+            else:
+                return x
+
+        df["pred"] = df["pred"].apply(_entailment_to_yes_no_maybe)
+        if single_answers:
+            df["gold"] = df["gold"].apply(_entailment_to_yes_no_maybe)
+        else:
+            df["gold"] = df["gold"].apply(
+                lambda x: [_entailment_to_yes_no_maybe(ans) for ans in x]
+            )
+
+    if bool_to_yes_no:
+
+        def map_bool_to_yes_no(x):
+            if isinstance(x, bool):
+                return "yes" if x else "no"
+            elif isinstance(x, str):
+                if x.lower() == "true":
+                    return "yes"
+                elif x.lower() == "false":
+                    return "no"
+            return x
+
+        df.pred = df.pred.apply(lambda x: str(map_bool_to_yes_no(x)).strip())
+        if single_answers:
+            df.gold = df.gold.apply(lambda ans: str(map_bool_to_yes_no(ans)).strip())
+        else:
+            df.gold = df.gold.apply(
+                lambda answers: [map_bool_to_yes_no(ans) for ans in answers]
+            )
+
+    df["pred_post_processed"] = vqa_clean(df["pred"].tolist())
+
+    scores: Dict[str, float] = {}
+    if single_answers:
+        scores["acc"] = (df["gold"] == df["pred"]).mean()
+        scores["relaxed_acc"] = df.apply(
+            lambda x: x["pred"].startswith(x["gold"]) or x["pred"].endswith(x["gold"]),
+            axis=1,
+        ).mean()
+
+        if vqa_post_process:
+            post_proc_acc = (df["gold"] == df["pred_post_processed"]).mean()
+            post_proc_relaxed_acc = df.apply(
+                lambda x: x["pred_post_processed"].startswith(x["gold"])
+                or x["pred_post_processed"].endswith(x["gold"]),
+                axis=1,
+            ).mean()
+            scores["acc_post_processed"] = post_proc_acc
+            scores["relaxed_acc_post_processed"] = post_proc_relaxed_acc
+    else:
+        scores["acc"] = df.apply(lambda x: x["pred"] in x["gold"], axis=1).mean()
+        scores["relaxed_acc"] = df.apply(
+            lambda x: any(
+                x["pred"].startswith(ans) or x["pred"].endswith(ans)
+                for ans in x["gold"]
+            ),
+            axis=1,
+        ).mean()
+
+        if vqa_post_process:
+            scores["acc_post_processed"] = df.apply(
+                lambda x: any(x["pred_post_processed"] == ans for ans in x["gold"]),
+                axis=1,
+            ).mean()
+            scores["relaxed_acc_post_processed"] = df.apply(
+                lambda x: any(
+                    x["pred_post_processed"].startswith(ans)
+                    or x["pred_post_processed"].endswith(ans)
+                    for ans in x["gold"]
+                ),
+                axis=1,
+            ).mean()
+
+    return scores
+
+
+# adapted from https://github.com/salesforce/LAVIS/blob/main/lavis/common/vqa_tools/vqa_eval.py
+def vqa_clean(labels: List[str]):
+    manualMap = {
+        "none": "0",
+        "zero": "0",
+        "one": "1",
+        "two": "2",
+        "three": "3",
+        "four": "4",
+        "five": "5",
+        "six": "6",
+        "seven": "7",
+        "eight": "8",
+        "nine": "9",
+        "ten": "10",
+    }
+    articles = ["a", "an", "the"]
+    periodStrip = re.compile(r"(?!<=\d)(\.)(?!\d)")
+    commaStrip = re.compile(r"(\d)(,)(\d)")
+    punct = [
+        ";",
+        r"/",
+        "[",
+        "]",
+        '"',
+        "{",
+        "}",
+        "(",
+        ")",
+        "=",
+        "+",
+        "\\",
+        "_",
+        "-",
+        ">",
+        "<",
+        "@",
+        "`",
+        ",",
+        "?",
+        "!",
+    ]
+    contractions = {
+        "aint": "ain't",
+        "arent": "aren't",
+        "cant": "can't",
+        "couldve": "could've",
+        "couldnt": "couldn't",
+        "couldn'tve": "couldn't've",
+        "couldnt've": "couldn't've",
+        "didnt": "didn't",
+        "doesnt": "doesn't",
+        "dont": "don't",
+        "hadnt": "hadn't",
+        "hadnt've": "hadn't've",
+        "hadn'tve": "hadn't've",
+        "hasnt": "hasn't",
+        "havent": "haven't",
+        "hed": "he'd",
+        "hed've": "he'd've",
+        "he'dve": "he'd've",
+        "hes": "he's",
+        "howd": "how'd",
+        "howll": "how'll",
+        "hows": "how's",
+        "Id've": "I'd've",
+        "I'dve": "I'd've",
+        "Im": "I'm",
+        "Ive": "I've",
+        "isnt": "isn't",
+        "itd": "it'd",
+        "itd've": "it'd've",
+        "it'dve": "it'd've",
+        "itll": "it'll",
+        "let's": "let's",
+        "maam": "ma'am",
+        "mightnt": "mightn't",
+        "mightnt've": "mightn't've",
+        "mightn'tve": "mightn't've",
+        "mightve": "might've",
+        "mustnt": "mustn't",
+        "mustve": "must've",
+        "neednt": "needn't",
+        "notve": "not've",
+        "oclock": "o'clock",
+        "oughtnt": "oughtn't",
+        "ow's'at": "'ow's'at",
+        "'ows'at": "'ow's'at",
+        "'ow'sat": "'ow's'at",
+        "shant": "shan't",
+        "shed've": "she'd've",
+        "she'dve": "she'd've",
+        "she's": "she's",
+        "shouldve": "should've",
+        "shouldnt": "shouldn't",
+        "shouldnt've": "shouldn't've",
+        "shouldn'tve": "shouldn't've",
+        "somebody'd": "somebodyd",
+        "somebodyd've": "somebody'd've",
+        "somebody'dve": "somebody'd've",
+        "somebodyll": "somebody'll",
+        "somebodys": "somebody's",
+        "someoned": "someone'd",
+        "someoned've": "someone'd've",
+        "someone'dve": "someone'd've",
+        "someonell": "someone'll",
+        "someones": "someone's",
+        "somethingd": "something'd",
+        "somethingd've": "something'd've",
+        "something'dve": "something'd've",
+        "somethingll": "something'll",
+        "thats": "that's",
+        "thered": "there'd",
+        "thered've": "there'd've",
+        "there'dve": "there'd've",
+        "therere": "there're",
+        "theres": "there's",
+        "theyd": "they'd",
+        "theyd've": "they'd've",
+        "they'dve": "they'd've",
+        "theyll": "they'll",
+        "theyre": "they're",
+        "theyve": "they've",
+        "twas": "'twas",
+        "wasnt": "wasn't",
+        "wed've": "we'd've",
+        "we'dve": "we'd've",
+        "weve": "we've",
+        "werent": "weren't",
+        "whatll": "what'll",
+        "whatre": "what're",
+        "whats": "what's",
+        "whatve": "what've",
+        "whens": "when's",
+        "whered": "where'd",
+        "wheres": "where's",
+        "whereve": "where've",
+        "whod": "who'd",
+        "whod've": "who'd've",
+        "who'dve": "who'd've",
+        "wholl": "who'll",
+        "whos": "who's",
+        "whove": "who've",
+        "whyll": "why'll",
+        "whyre": "why're",
+        "whys": "why's",
+        "wont": "won't",
+        "wouldve": "would've",
+        "wouldnt": "wouldn't",
+        "wouldnt've": "wouldn't've",
+        "wouldn'tve": "wouldn't've",
+        "yall": "y'all",
+        "yall'll": "y'all'll",
+        "y'allll": "y'all'll",
+        "yall'd've": "y'all'd've",
+        "y'alld've": "y'all'd've",
+        "y'all'dve": "y'all'd've",
+        "youd": "you'd",
+        "youd've": "you'd've",
+        "you'dve": "you'd've",
+        "youll": "you'll",
+        "youre": "you're",
+        "youve": "you've",
+    }
+
+    def processPunctuation(inText):
+        outText = inText
+        for p in punct:
+            if (p + " " in inText or " " + p in inText) or (
+                re.search(commaStrip, inText) is not None
+            ):
+                outText = outText.replace(p, "")
+            else:
+                outText = outText.replace(p, " ")
+        outText = periodStrip.sub("", outText, re.UNICODE)
+        return outText
+
+    def processDigitArticle(inText):
+        outText = []
+        tempText = inText.lower().split()
+        for word in tempText:
+            word = manualMap.setdefault(word, word)
+            if word not in articles:
+                outText.append(word)
+            else:
+                pass
+        for wordId, word in enumerate(outText):
+            if word in contractions:
+                outText[wordId] = contractions[word]
+        outText = " ".join(outText)
+        return outText
+
+    cleaned_labels = []
+    for label in labels:
+        label = label.replace("\n", "").replace("\t", "").strip()
+        label = processPunctuation(label)
+        label = processDigitArticle(label)
+        cleaned_labels.append(label)
+    return cleaned_labels
+
+
+chair_coco_object_synonyms = [
+    [
+        "person",
+        "girl",
+        "boy",
+        "man",
+        "woman",
+        "kid",
+        "child",
+        "chef",
+        "baker",
+        "people",
+        "adult",
+        "rider",
+        "children",
+        "baby",
+        "worker",
+        "passenger",
+        "sister",
+        "biker",
+        "policeman",
+        "cop",
+        "officer",
+        "lady",
+        "cowboy",
+        "bride",
+        "groom",
+        "male",
+        "female",
+        "guy",
+        "traveler",
+        "mother",
+        "father",
+        "gentleman",
+        "pitcher",
+        "player",
+        "skier",
+        "snowboarder",
+        "skater",
+        "skateboarder",
+        "person",
+        "woman",
+        "guy",
+        "foreigner",
+        "child",
+        "gentleman",
+        "caller",
+        "offender",
+        "coworker",
+        "trespasser",
+        "patient",
+        "politician",
+        "soldier",
+        "grandchild",
+        "serviceman",
+        "walker",
+        "drinker",
+        "doctor",
+        "bicyclist",
+        "thief",
+        "buyer",
+        "teenager",
+        "student",
+        "camper",
+        "driver",
+        "solider",
+        "hunter",
+        "shopper",
+        "villager",
+    ],
+    ["bicycle", "bike", "bicycle", "bike", "unicycle", "minibike", "trike"],
+    [
+        "car",
+        "automobile",
+        "van",
+        "minivan",
+        "sedan",
+        "suv",
+        "hatchback",
+        "cab",
+        "jeep",
+        "coupe",
+        "taxicab",
+        "limo",
+        "taxi",
+    ],
+    [
+        "motorcycle",
+        "scooter",
+        " motor bike",
+        "motor cycle",
+        "motorbike",
+        "scooter",
+        "moped",
+    ],
+    [
+        "airplane",
+        "jetliner",
+        "plane",
+        "air plane",
+        "monoplane",
+        "aircraft",
+        "jet",
+        "jetliner",
+        "airbus",
+        "biplane",
+        "seaplane",
+    ],
+    ["bus", "minibus", "trolley"],
+    ["train", "locomotive", "tramway", "caboose"],
+    ["truck", "pickup", "lorry", "hauler", "firetruck"],
+    [
+        "boat",
+        "ship",
+        "liner",
+        "sailboat",
+        "motorboat",
+        "dinghy",
+        "powerboat",
+        "speedboat",
+        "canoe",
+        "skiff",
+        "yacht",
+        "kayak",
+        "catamaran",
+        "pontoon",
+        "houseboat",
+        "vessel",
+        "rowboat",
+        "trawler",
+        "ferryboat",
+        "watercraft",
+        "tugboat",
+        "schooner",
+        "barge",
+        "ferry",
+        "sailboard",
+        "paddleboat",
+        "lifeboat",
+        "freighter",
+        "steamboat",
+        "riverboat",
+        "battleship",
+        "steamship",
+    ],
+    [
+        "traffic light",
+        "street light",
+        "traffic signal",
+        "stop light",
+        "streetlight",
+        "stoplight",
+    ],
+    ["fire hydrant", "hydrant"],
+    ["stop sign"],
+    ["parking meter"],
+    ["bench", "pew"],
+    [
+        "bird",
+        "ostrich",
+        "owl",
+        "seagull",
+        "goose",
+        "duck",
+        "parakeet",
+        "falcon",
+        "robin",
+        "pelican",
+        "waterfowl",
+        "heron",
+        "hummingbird",
+        "mallard",
+        "finch",
+        "pigeon",
+        "sparrow",
+        "seabird",
+        "osprey",
+        "blackbird",
+        "fowl",
+        "shorebird",
+        "woodpecker",
+        "egret",
+        "chickadee",
+        "quail",
+        "bluebird",
+        "kingfisher",
+        "buzzard",
+        "willet",
+        "gull",
+        "swan",
+        "bluejay",
+        "flamingo",
+        "cormorant",
+        "parrot",
+        "loon",
+        "gosling",
+        "waterbird",
+        "pheasant",
+        "rooster",
+        "sandpiper",
+        "crow",
+        "raven",
+        "turkey",
+        "oriole",
+        "cowbird",
+        "warbler",
+        "magpie",
+        "peacock",
+        "cockatiel",
+        "lorikeet",
+        "puffin",
+        "vulture",
+        "condor",
+        "macaw",
+        "peafowl",
+        "cockatoo",
+        "songbird",
+    ],
+    ["cat", "kitten", "feline", "tabby"],
+    [
+        "dog",
+        "puppy",
+        "beagle",
+        "pup",
+        "chihuahua",
+        "schnauzer",
+        "dachshund",
+        "rottweiler",
+        "canine",
+        "pitbull",
+        "collie",
+        "pug",
+        "terrier",
+        "poodle",
+        "labrador",
+        "doggie",
+        "doberman",
+        "mutt",
+        "doggy",
+        "spaniel",
+        "bulldog",
+        "sheepdog",
+        "weimaraner",
+        "corgi",
+        "cocker",
+        "greyhound",
+        "retriever",
+        "brindle",
+        "hound",
+        "whippet",
+        "husky",
+    ],
+    [
+        "horse",
+        "colt",
+        "pony",
+        "racehorse",
+        "stallion",
+        "equine",
+        "mare",
+        "foal",
+        "palomino",
+        "mustang",
+        "clydesdale",
+        "bronc",
+        "bronco",
+    ],
+    ["sheep", "lamb", "ram", "lamb", "goat", "ewe"],
+    [
+        "cow",
+        "cattle",
+        "oxen",
+        "ox",
+        "calf",
+        "cattle",
+        "holstein",
+        "heifer",
+        "buffalo",
+        "bull",
+        "zebu",
+        "bison",
+    ],
+    ["elephant"],
+    ["bear", "panda"],
+    ["zebra"],
+    ["giraffe"],
+    ["backpack", "knapsack"],
+    ["umbrella"],
+    ["handbag", "wallet", "purse", "briefcase"],
+    ["tie", "bow", "bow tie"],
+    ["suitcase", "suit case", "luggage"],
+    ["frisbee"],
+    ["skis", "ski"],
+    ["snowboard"],
+    ["sports ball", "ball"],
+    ["kite"],
+    ["baseball bat"],
+    ["baseball glove"],
+    ["skateboard"],
+    ["surfboard", "longboard", "skimboard", "shortboard", "wakeboard"],
+    ["tennis racket", "racket"],
+    ["bottle"],
+    ["wine glass"],
+    ["cup"],
+    ["fork"],
+    ["knife", "pocketknife", "knive"],
+    ["spoon"],
+    ["bowl", "container"],
+    ["banana"],
+    ["apple"],
+    ["sandwich", "burger", "sub", "cheeseburger", "hamburger"],
+    ["orange"],
+    ["broccoli"],
+    ["carrot"],
+    ["hot dog"],
+    ["pizza"],
+    ["donut", "doughnut", "bagel"],
+    ["cake", " cheesecake", "cupcake", "shortcake", "coffeecake", "pancake"],
+    ["chair", "seat", "stool"],
+    ["couch", "sofa", "recliner", "futon", "loveseat", "settee", "chesterfield"],
+    ["potted plant", "houseplant"],
+    ["bed"],
+    ["dining table", "table", "desk"],
+    ["toilet", "urinal", "commode", "toilet", "lavatory", "potty"],
+    ["tv", "monitor", "televison", "television"],
+    [
+        "laptop",
+        "computer",
+        "notebook",
+        "netbook",
+        "lenovo",
+        "macbook",
+        "laptop computer",
+    ],
+    ["mouse"],
+    ["remote"],
+    ["keyboard"],
+    [
+        "cell phone",
+        "mobile phone",
+        "phone",
+        "cellphone",
+        "telephone",
+        "phon",
+        "smartphone",
+        "iPhone",
+    ],
+    ["microwave"],
+    ["oven", "stovetop", "stove", "stove top oven"],
+    ["toaster"],
+    ["sink"],
+    ["refrigerator", "fridge", "fridge", "freezer"],
+    ["book"],
+    ["clock"],
+    ["vase"],
+    ["scissors"],
+    ["teddy bear", "teddybear"],
+    ["hair drier", "hairdryer"],
+    ["toothbrush"],
+]
+
+
+def chair(image_ids, text_labels, captions, print_examples=10):
+    import nltk
+
+    nltk.download("punkt")
+    # nltk.download('omw-1.4')
+
+    image_ids = [id for ids in image_ids for id in ids]
+    text_labels = [label for labels in text_labels for label in labels]
+    captions = [c for caps in captions for c in caps]
+
+    print(f"Average characters: {sum(len(c) for c in captions)/len(captions)}")
+
+    synonyms = chair_coco_object_synonyms
+    mscoco_objects = []  # mscoco objects and *all* synonyms
+    inverse_synonym_dict = {}
+    for synonym in synonyms:
+        mscoco_objects.extend(synonym)
+        for s in synonym:
+            inverse_synonym_dict[s] = synonym[0]
+
+    # Some hard coded rules for implementing CHAIR metrics on MSCOCO
+
+    # common 'double words' in MSCOCO that should be treated as a single word
+    coco_double_words = [
+        "motor bike",
+        "motor cycle",
+        "air plane",
+        "traffic light",
+        "street light",
+        "traffic signal",
+        "stop light",
+        "fire hydrant",
+        "stop sign",
+        "parking meter",
+        "suit case",
+        "sports ball",
+        "baseball bat",
+        "baseball glove",
+        "tennis racket",
+        "wine glass",
+        "hot dog",
+        "cell phone",
+        "mobile phone",
+        "teddy bear",
+        "hair drier",
+        "potted plant",
+        "bow tie",
+        "laptop computer",
+        "stove top oven",
+        "hot dog",
+        "teddy bear",
+        "home plate",
+        "train track",
+    ]
+
+    # Hard code some rules for special cases in MSCOCO
+    # qualifiers like 'baby' or 'adult' animal will lead to a false fire for the MSCOCO object 'person'.  'baby bird' --> 'bird'.
+    animal_words = [
+        "bird",
+        "cat",
+        "dog",
+        "horse",
+        "sheep",
+        "cow",
+        "elephant",
+        "bear",
+        "zebra",
+        "giraffe",
+        "animal",
+        "cub",
+    ]
+    # qualifiers like 'passenger' vehicle will lead to a false fire for the MSCOCO object 'person'.  'passenger jet' --> 'jet'.
+    vehicle_words = ["jet", "train"]
+
+    # double_word_dict will map double words to the word they should be treated as in our analysis
+
+    double_word_dict = {}
+    for double_word in coco_double_words:
+        double_word_dict[double_word] = double_word
+    for animal_word in animal_words:
+        double_word_dict["baby %s" % animal_word] = animal_word
+        double_word_dict["adult %s" % animal_word] = animal_word
+    for vehicle_word in vehicle_words:
+        double_word_dict["passenger %s" % vehicle_word] = vehicle_word
+    double_word_dict["bow tie"] = "tie"
+    double_word_dict["toilet seat"] = "toilet"
+    double_word_dict["wine glas"] = "wine glass"
+
+    def caption_to_words(caption):
+        """
+        Input: caption
+        Output: MSCOCO words in the caption
+        """
+
+        # standard preprocessing
+        words = nltk.word_tokenize(caption.lower())
+        words = [singularize(w) for w in words]
+
+        # replace double words
+        i = 0
+        double_words = []
+        idxs = []
+        while i < len(words):
+            idxs.append(i)
+            double_word = " ".join(words[i : i + 2])
+            if double_word in double_word_dict:
+                double_words.append(double_word_dict[double_word])
+                i += 2
+            else:
+                double_words.append(words[i])
+                i += 1
+        words = double_words
+
+        # toilet seat is not chair (sentences like "the seat of the toilet" will fire for "chair" if we do not include this line)
+        if ("toilet" in words) & ("seat" in words):
+            words = [word for word in words if word != "seat"]
+
+        # get synonyms for all words in the caption
+        idxs = [
+            idxs[idx] for idx, word in enumerate(words) if word in set(mscoco_objects)
+        ]
+        words = [word for word in words if word in set(mscoco_objects)]
+        node_words = []
+        for word in words:
+            node_words.append(inverse_synonym_dict[word])
+        # return all the MSCOCO objects in the caption
+        return words, node_words, idxs, double_words
+
+    num_caps = 0.0
+    num_hallucinated_caps = 0.0
+    hallucinated_word_count = 0.0
+    coco_word_count = 0.0
+
+    for i, (iid, cap, gt_objects) in enumerate(zip(image_ids, captions, text_labels)):
+        # get all words in the caption, as well as corresponding node word
+        words, node_words, idxs, raw_words = caption_to_words(cap)
+
+        # count hallucinated words
+        coco_word_count += len(node_words)
+        hallucinated = False
+        mscoco_hallucinated_words = []
+        for word, node_word, idx in zip(words, node_words, idxs):
+            if node_word not in gt_objects:
+                hallucinated_word_count += 1
+                mscoco_hallucinated_words.append((word, node_word))
+                # cap_dict['hallucination_idxs'].append(idx)
+                hallucinated = True
+        if print_examples > 0 and i < print_examples:
+            print(
+                f"{iid} -- {cap} -- {words} {node_words} -- {mscoco_hallucinated_words}"
+            )
+        # count hallucinated caps
+        num_caps += 1
+        if hallucinated:
+            num_hallucinated_caps += 1
+
+        # cap_dict['metrics']['CHAIRs'] = int(hallucinated)
+        # cap_dict['metrics']['CHAIRi'] = 0.
+        # if len(words) > 0:
+        #     cap_dict['metrics']['CHAIRi'] = len(cap_dict['mscoco_hallucinated_words']) / float(len(words))
+
+    chair_s = num_hallucinated_caps / num_caps
+    chair_i = hallucinated_word_count / coco_word_count
+
+    return dict(chair_s=chair_s, chair_i=chair_i)
+
+
+#### SINGULARIZE ###################################################################################
+# Adapted from Bermi Ferrer's Inflector for Python:
+# http://www.bermi.org/inflector/
+
+# Copyright (c) 2006 Bermi Ferrer Martinez
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software to deal in this software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of this software, and to permit
+# persons to whom this software is furnished to do so, subject to the following
+# condition:
+#
+# THIS SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THIS SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THIS SOFTWARE.
+
+singular_rules = [
+    (r"(?i)(.)ae$", "\\1a"),
+    (r"(?i)(.)itis$", "\\1itis"),
+    (r"(?i)(.)eaux$", "\\1eau"),
+    (r"(?i)(quiz)zes$", "\\1"),
+    (r"(?i)(matr)ices$", "\\1ix"),
+    (r"(?i)(ap|vert|ind)ices$", "\\1ex"),
+    (r"(?i)^(ox)en", "\\1"),
+    (r"(?i)(alias|status)es$", "\\1"),
+    (r"(?i)([octop|vir])i$", "\\1us"),
+    (r"(?i)(cris|ax|test)es$", "\\1is"),
+    (r"(?i)(shoe)s$", "\\1"),
+    (r"(?i)(o)es$", "\\1"),
+    (r"(?i)(bus)es$", "\\1"),
+    (r"(?i)([m|l])ice$", "\\1ouse"),
+    (r"(?i)(x|ch|ss|sh)es$", "\\1"),
+    (r"(?i)(m)ovies$", "\\1ovie"),
+    (r"(?i)(.)ombies$", "\\1ombie"),
+    (r"(?i)(s)eries$", "\\1eries"),
+    (r"(?i)([^aeiouy]|qu)ies$", "\\1y"),
+    # -f, -fe sometimes take -ves in the plural
+    # (e.g., lives, wolves).
+    (r"([aeo]l)ves$", "\\1f"),
+    (r"([^d]ea)ves$", "\\1f"),
+    (r"arves$", "arf"),
+    (r"erves$", "erve"),
+    (r"([nlw]i)ves$", "\\1fe"),
+    (r"(?i)([lr])ves$", "\\1f"),
+    (r"([aeo])ves$", "\\1ve"),
+    (r"(?i)(sive)s$", "\\1"),
+    (r"(?i)(tive)s$", "\\1"),
+    (r"(?i)(hive)s$", "\\1"),
+    (r"(?i)([^f])ves$", "\\1fe"),
+    # -ses suffixes.
+    (r"(?i)(^analy)ses$", "\\1sis"),
+    (r"(?i)((a)naly|(b)a|(d)iagno|(p)arenthe|(p)rogno|(s)ynop|(t)he)ses$", "\\1\\2sis"),
+    (r"(?i)(.)opses$", "\\1opsis"),
+    (r"(?i)(.)yses$", "\\1ysis"),
+    (r"(?i)(h|d|r|o|n|b|cl|p)oses$", "\\1ose"),
+    (r"(?i)(fruct|gluc|galact|lact|ket|malt|rib|sacchar|cellul)ose$", "\\1ose"),
+    (r"(?i)(.)oses$", "\\1osis"),
+    # -a
+    (r"(?i)([ti])a$", "\\1um"),
+    (r"(?i)(n)ews$", "\\1ews"),
+    (r"(?i)s$", ""),
+]
+
+# For performance, compile the regular expressions only once:
+singular_rules = [(re.compile(r[0]), r[1]) for r in singular_rules]
+
+singular_uninflected = set(
+    (
+        "bison",
+        "debris",
+        "headquarters",
+        "pincers",
+        "trout",
+        "bream",
+        "diabetes",
+        "herpes",
+        "pliers",
+        "tuna",
+        "breeches",
+        "djinn",
+        "high-jinks",
+        "proceedings",
+        "whiting",
+        "britches",
+        "eland",
+        "homework",
+        "rabies",
+        "wildebeest",
+        "carp",
+        "elk",
+        "innings",
+        "salmon",
+        "chassis",
+        "flounder",
+        "jackanapes",
+        "scissors",
+        "christmas",
+        "gallows",
+        "mackerel",
+        "series",
+        "clippers",
+        "georgia",
+        "measles",
+        "shears",
+        "cod",
+        "graffiti",
+        "mews",
+        "species",
+        "contretemps",
+        "mumps",
+        "swine",
+        "corps",
+        "news",
+        "swiss",
+    )
+)
+singular_uncountable = set(
+    (
+        "advice",
+        "equipment",
+        "happiness",
+        "luggage",
+        "news",
+        "software",
+        "bread",
+        "fruit",
+        "information",
+        "mathematics",
+        "progress",
+        "understanding",
+        "butter",
+        "furniture",
+        "ketchup",
+        "mayonnaise",
+        "research",
+        "water",
+        "cheese",
+        "garbage",
+        "knowledge",
+        "meat",
+        "rice",
+        "electricity",
+        "gravel",
+        "love",
+        "mustard",
+        "sand",
+    )
+)
+singular_ie = set(
+    (
+        "alergie",
+        "cutie",
+        "hoagie",
+        "newbie",
+        "softie",
+        "veggie",
+        "auntie",
+        "doggie",
+        "hottie",
+        "nightie",
+        "sortie",
+        "weenie",
+        "beanie",
+        "eyrie",
+        "indie",
+        "oldie",
+        "stoolie",
+        "yuppie",
+        "birdie",
+        "freebie",
+        "junkie",
+        "^pie",
+        "sweetie",
+        "zombie",
+        "bogie",
+        "goonie",
+        "laddie",
+        "pixie",
+        "techie",
+        "bombie",
+        "groupie",
+        "laramie",
+        "quickie",
+        "^tie",
+        "collie",
+        "hankie",
+        "lingerie",
+        "reverie",
+        "toughie",
+        "cookie",
+        "hippie",
+        "meanie",
+        "rookie",
+        "valkyrie",
+    )
+)
+singular_irregular = {
+    "atlantes": "atlas",
+    "atlases": "atlas",
+    "axes": "axe",
+    "beeves": "beef",
+    "brethren": "brother",
+    "children": "child",
+    "corpora": "corpus",
+    "corpuses": "corpus",
+    "ephemerides": "ephemeris",
+    "feet": "foot",
+    "ganglia": "ganglion",
+    "geese": "goose",
+    "genera": "genus",
+    "genii": "genie",
+    "graffiti": "graffito",
+    "helves": "helve",
+    "kine": "cow",
+    "leaves": "leaf",
+    "loaves": "loaf",
+    "men": "man",
+    "mongooses": "mongoose",
+    "monies": "money",
+    "moves": "move",
+    "mythoi": "mythos",
+    "numena": "numen",
+    "occipita": "occiput",
+    "octopodes": "octopus",
+    "opera": "opus",
+    "opuses": "opus",
+    "our": "my",
+    "oxen": "ox",
+    "penes": "penis",
+    "penises": "penis",
+    "people": "person",
+    "sexes": "sex",
+    "soliloquies": "soliloquy",
+    "teeth": "tooth",
+    "testes": "testis",
+    "trilbys": "trilby",
+    "turves": "turf",
+    "zoa": "zoon",
+}
+
+plural_prepositions = set(
+    (
+        "about",
+        "before",
+        "during",
+        "of",
+        "till",
+        "above",
+        "behind",
+        "except",
+        "off",
+        "to",
+        "across",
+        "below",
+        "for",
+        "on",
+        "under",
+        "after",
+        "beneath",
+        "from",
+        "onto",
+        "until",
+        "among",
+        "beside",
+        "in",
+        "out",
+        "unto",
+        "around",
+        "besides",
+        "into",
+        "over",
+        "upon",
+        "at",
+        "between",
+        "near",
+        "since",
+        "with",
+        "athwart",
+        "betwixt",
+        "beyond",
+        "but",
+        "by",
+    )
+)
+
+VERB, NOUN, ADJECTIVE, ADVERB = "VB", "NN", "JJ", "RB"
+
+
+def singularize(word, pos=NOUN, custom={}):
+    """Returns the singular of a given word."""
+    if word in custom:
+        return custom[word]
+    # Recurse compound words (e.g. mothers-in-law).
+    if "-" in word:
+        w = word.split("-")
+        if len(w) > 1 and w[1] in plural_prepositions:
+            return singularize(w[0], pos, custom) + "-" + "-".join(w[1:])
+    # dogs' => dog's
+    if word.endswith("'"):
+        return singularize(word[:-1]) + "'s"
+    w = word.lower()
+    for x in singular_uninflected:
+        if x.endswith(w):
+            return word
+    for x in singular_uncountable:
+        if x.endswith(w):
+            return word
+    for x in singular_ie:
+        if w.endswith(x + "s"):
+            return w
+    for x in singular_irregular:
+        if w.endswith(x):
+            return re.sub("(?i)" + x + "$", singular_irregular[x], word)
+    for suffix, inflection in singular_rules:
+        m = suffix.search(word)
+        g = m and m.groups() or []
+        if m:
+            for k in range(len(g)):
+                if g[k] is None:
+                    inflection = inflection.replace("\\" + str(k + 1), "")
+            return suffix.sub(inflection, word)
+    return word
diff --git a/lmms_eval/tasks/maxm/maxm.yaml b/lmms_eval/tasks/maxm/maxm.yaml
new file mode 100755
index 000000000..22f9a4b1e
--- /dev/null
+++ b/lmms_eval/tasks/maxm/maxm.yaml
@@ -0,0 +1,9 @@
+group : maxm
+task:
+  - maxm_en
+  - maxm_fr
+  - maxm_hi
+  - maxm_iw
+  - maxm_ro
+  - maxm_th
+  - maxm_zh
\ No newline at end of file
diff --git a/lmms_eval/tasks/maxm/maxm_en.yaml b/lmms_eval/tasks/maxm/maxm_en.yaml
new file mode 100755
index 000000000..0e159b949
--- /dev/null
+++ b/lmms_eval/tasks/maxm/maxm_en.yaml
@@ -0,0 +1,57 @@
+dataset_path: neulab/PangeaBench-maxm
+dataset_kwargs:
+  token: True
+task: "maxm_en"
+# group: "maxm"
+test_split: en
+output_type: generate_until
+doc_to_visual: !function utils.maxm_doc_to_visual
+doc_to_text: !function utils.maxm_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 64
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.maxm_process_results
+metric_list:
+  - metric: exact_match
+    aggregation: !function utils.maxm_ema
+    higher_is_better: true
+  - metric: rouge_l
+    aggregation: !function utils.maxm_rouge_l
+    higher_is_better: true
+  - metric: cider
+    aggregation: !function utils.maxm_cider
+    higher_is_better: true
+  - metric: relaxed_accuracy
+    aggregation: !function utils.maxm_relaxed_ema
+    higher_is_better: true
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in English."
+  llava:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in English."
+  mblip:
+    pre_prompt: "Question: "
+    post_prompt: "Answer in a single word or phrase in English:"
+  paligemma:
+    pre_prompt: "answer en"
+    post_prompt: "\nAnswer the question using a single word or phrase in English."
+  palo_7b:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in English."
+  cambrian1:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in English."
+  gpt4v:
+    pre_prompt: "Analyze the image and answer the question concisely. "
+    post_prompt: ""
+  qwen_vl:  
+    pre_prompt: "Based on the image, answer the following question: "
+    post_prompt: " Answer:"
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/maxm/maxm_fr.yaml b/lmms_eval/tasks/maxm/maxm_fr.yaml
new file mode 100755
index 000000000..26592f56d
--- /dev/null
+++ b/lmms_eval/tasks/maxm/maxm_fr.yaml
@@ -0,0 +1,57 @@
+dataset_path: neulab/PangeaBench-maxm
+dataset_kwargs:
+  token: True
+task: "maxm_fr"
+# group: "maxm"
+test_split: fr
+output_type: generate_until
+doc_to_visual: !function utils.maxm_doc_to_visual
+doc_to_text: !function utils.maxm_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 64
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.maxm_process_results
+metric_list:
+  - metric: exact_match
+    aggregation: !function utils.maxm_ema
+    higher_is_better: true
+  - metric: rouge_l
+    aggregation: !function utils.maxm_rouge_l
+    higher_is_better: true
+  - metric: cider
+    aggregation: !function utils.maxm_cider
+    higher_is_better: true
+  - metric: relaxed_accuracy
+    aggregation: !function utils.maxm_relaxed_ema
+    higher_is_better: true
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in French."
+  llava:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in French."
+  mblip:
+    pre_prompt: "Question: "
+    post_prompt: "Answer in a single word or phrase in French:"
+  paligemma:
+    pre_prompt: "answer fr"
+    post_prompt: "\nAnswer the question using a single word or phrase in French."
+  palo_7b:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in French."
+  cambrian1:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in French."
+  gpt4v:
+    pre_prompt: "Analyze the image and answer the question concisely. "
+    post_prompt: ""
+  qwen_vl:  
+    pre_prompt: "Based on the image, answer the following question: "
+    post_prompt: " Answer:"
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/maxm/maxm_hi.yaml b/lmms_eval/tasks/maxm/maxm_hi.yaml
new file mode 100755
index 000000000..7d4fde6e9
--- /dev/null
+++ b/lmms_eval/tasks/maxm/maxm_hi.yaml
@@ -0,0 +1,57 @@
+dataset_path: neulab/PangeaBench-maxm
+dataset_kwargs:
+  token: True
+task: "maxm_hi"
+group: "maxm"
+test_split: hi
+output_type: generate_until
+doc_to_visual: !function utils.maxm_doc_to_visual
+doc_to_text: !function utils.maxm_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 64
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.maxm_process_results
+metric_list:
+  - metric: exact_match
+    aggregation: !function utils.maxm_ema
+    higher_is_better: true
+  - metric: rouge_l
+    aggregation: !function utils.maxm_rouge_l
+    higher_is_better: true
+  - metric: cider
+    aggregation: !function utils.maxm_cider
+    higher_is_better: true
+  - metric: relaxed_accuracy
+    aggregation: !function utils.maxm_relaxed_ema
+    higher_is_better: true
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in Hindi."
+  llava:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in Hindi."
+  mblip:
+    pre_prompt: "Question: "
+    post_prompt: "Answer in a single word or phrase in Hindi:"
+  paligemma:
+    pre_prompt: "answer hi"
+    post_prompt: "\nAnswer the question using a single word or phrase in Hindi."
+  palo_7b:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in Hindi."
+  cambrian1:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in Hindi."
+  gpt4v:
+    pre_prompt: "Analyze the image and answer the question concisely. "
+    post_prompt: ""
+  qwen_vl:  
+    pre_prompt: "Based on the image, answer the following question: "
+    post_prompt: " Answer:"
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/maxm/maxm_iw.yaml b/lmms_eval/tasks/maxm/maxm_iw.yaml
new file mode 100755
index 000000000..be25cb772
--- /dev/null
+++ b/lmms_eval/tasks/maxm/maxm_iw.yaml
@@ -0,0 +1,57 @@
+dataset_path: neulab/PangeaBench-maxm
+dataset_kwargs:
+  token: True
+task: "maxm_iw"
+group: "maxm"
+test_split: iw
+output_type: generate_until
+doc_to_visual: !function utils.maxm_doc_to_visual
+doc_to_text: !function utils.maxm_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 64
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.maxm_process_results
+metric_list:
+  - metric: exact_match
+    aggregation: !function utils.maxm_ema
+    higher_is_better: true
+  - metric: rouge_l
+    aggregation: !function utils.maxm_rouge_l
+    higher_is_better: true
+  - metric: cider
+    aggregation: !function utils.maxm_cider
+    higher_is_better: true
+  - metric: relaxed_accuracy
+    aggregation: !function utils.maxm_relaxed_ema
+    higher_is_better: true
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in Hebrew."
+  llava:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in Hebrew."
+  mblip:
+    pre_prompt: "Question: "
+    post_prompt: "Answer in a single word or phrase in Hebrew:"
+  paligemma:
+    pre_prompt: "answer iw"
+    post_prompt: "\nAnswer the question using a single word or phrase in Hebrew."
+  palo_7b:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in Hebrew."
+  cambrian1:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in Hebrew."
+  gpt4v:
+    pre_prompt: "Analyze the image and answer the question concisely. "
+    post_prompt: ""
+  qwen_vl:  
+    pre_prompt: "Based on the image, answer the following question: "
+    post_prompt: " Answer:"
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/maxm/maxm_ro.yaml b/lmms_eval/tasks/maxm/maxm_ro.yaml
new file mode 100755
index 000000000..96ed424f8
--- /dev/null
+++ b/lmms_eval/tasks/maxm/maxm_ro.yaml
@@ -0,0 +1,57 @@
+dataset_path: neulab/PangeaBench-maxm
+dataset_kwargs:
+  token: True
+task: "maxm_ro"
+group: "maxm"
+test_split: ro
+output_type: generate_until
+doc_to_visual: !function utils.maxm_doc_to_visual
+doc_to_text: !function utils.maxm_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 64
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.maxm_process_results
+metric_list:
+  - metric: exact_match
+    aggregation: !function utils.maxm_ema
+    higher_is_better: true
+  - metric: rouge_l
+    aggregation: !function utils.maxm_rouge_l
+    higher_is_better: true
+  - metric: cider
+    aggregation: !function utils.maxm_cider
+    higher_is_better: true
+  - metric: relaxed_accuracy
+    aggregation: !function utils.maxm_relaxed_ema
+    higher_is_better: true
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in Romanian."
+  llava:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in Romanian."
+  mblip:
+    pre_prompt: "Question: "
+    post_prompt: "Answer in a single word or phrase in Romanian:"
+  paligemma:
+    pre_prompt: "answer ro"
+    post_prompt: "\nAnswer the question using a single word or phrase in Romanian."
+  palo_7b:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in Romanian."
+  cambrian1:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in Romanian."
+  gpt4v:
+    pre_prompt: "Analyze the image and answer the question concisely. "
+    post_prompt: ""
+  qwen_vl:  
+    pre_prompt: "Based on the image, answer the following question: "
+    post_prompt: " Answer:"
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/maxm/maxm_th.yaml b/lmms_eval/tasks/maxm/maxm_th.yaml
new file mode 100755
index 000000000..d0935c6b2
--- /dev/null
+++ b/lmms_eval/tasks/maxm/maxm_th.yaml
@@ -0,0 +1,57 @@
+dataset_path: neulab/PangeaBench-maxm
+dataset_kwargs:
+  token: True
+task: "maxm_th"
+group: "maxm"
+test_split: th
+output_type: generate_until
+doc_to_visual: !function utils.maxm_doc_to_visual
+doc_to_text: !function utils.maxm_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 64
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.maxm_process_results
+metric_list:
+  - metric: exact_match
+    aggregation: !function utils.maxm_ema
+    higher_is_better: true
+  - metric: rouge_l
+    aggregation: !function utils.maxm_rouge_l
+    higher_is_better: true
+  - metric: cider
+    aggregation: !function utils.maxm_cider
+    higher_is_better: true
+  - metric: relaxed_accuracy
+    aggregation: !function utils.maxm_relaxed_ema
+    higher_is_better: true
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in Thai."
+  llava:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in Thai."
+  mblip:
+    pre_prompt: "Question: "
+    post_prompt: "Answer in a single word or phrase in Thai:"
+  paligemma:
+    pre_prompt: "answer th"
+    post_prompt: "\nAnswer the question using a single word or phrase in Thai."
+  palo_7b:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in Thai."
+  cambrian1:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in Thai."
+  gpt4v:
+    pre_prompt: "Analyze the image and answer the question concisely. "
+    post_prompt: ""
+  qwen_vl:  
+    pre_prompt: "Based on the image, answer the following question: "
+    post_prompt: " Answer:"
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/maxm/maxm_zh.yaml b/lmms_eval/tasks/maxm/maxm_zh.yaml
new file mode 100755
index 000000000..599a247a9
--- /dev/null
+++ b/lmms_eval/tasks/maxm/maxm_zh.yaml
@@ -0,0 +1,57 @@
+dataset_path: neulab/PangeaBench-maxm
+dataset_kwargs:
+  token: True
+task: "maxm_zh"
+group: "maxm"
+test_split: zh
+output_type: generate_until
+doc_to_visual: !function utils.maxm_doc_to_visual
+doc_to_text: !function utils.maxm_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 64
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.maxm_process_results
+metric_list:
+  - metric: exact_match
+    aggregation: !function utils.maxm_ema
+    higher_is_better: true
+  - metric: rouge_l
+    aggregation: !function utils.maxm_rouge_l
+    higher_is_better: true
+  - metric: cider
+    aggregation: !function utils.maxm_cider
+    higher_is_better: true
+  - metric: relaxed_accuracy
+    aggregation: !function utils.maxm_relaxed_ema
+    higher_is_better: true
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in Chinese."
+  llava:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in Chinese."
+  mblip:
+    pre_prompt: "Question: "
+    post_prompt: "Answer in a single word or phrase in Chinese:"
+  paligemma:
+    pre_prompt: "answer zh"
+    post_prompt: "\nAnswer the question using a single word or phrase in Chinese."
+  palo_7b:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in Chinese."
+  cambrian1:
+    pre_prompt: "Given the image and the question, provide a concise answer. "
+    post_prompt: "\nAnswer the question using a single word or phrase in Chinese."
+  gpt4v:
+    pre_prompt: "Analyze the image and answer the question concisely. "
+    post_prompt: ""
+  qwen_vl:  
+    pre_prompt: "Based on the image, answer the following question: "
+    post_prompt: " Answer:"
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/maxm/utils.py b/lmms_eval/tasks/maxm/utils.py
new file mode 100755
index 000000000..59c909dd8
--- /dev/null
+++ b/lmms_eval/tasks/maxm/utils.py
@@ -0,0 +1,566 @@
+from collections import defaultdict
+import os
+import datetime
+import json
+from typing import Dict, List, Literal
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+from datasets import Image as DatasetsImage
+from pycocoevalcap.eval import COCOEvalCap, Rouge, Cider
+from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer
+from pycocotools.coco import COCO
+
+import unicodedata
+import re
+
+import pandas as pd
+
+import os
+from loguru import logger as eval_logger
+
+
+
+# Example usage with extended synonyms for English, French, Hindi, Hebrew, Romanian, Thai, and Chinese
+synonyms = {
+    "true": ["yes", "1", "true", "vrai", "हां", "נכון", "adevărat", "ใช่", "是"],
+    "false": ["no", "0", "false", "faux", "नहीं", "לא נכון", "fals", "ไม่ใช่", "否"],
+    "yes": ["true", "1", "yes", "oui", "हां", "כן", "da", "ใช่", "是"],
+    "no": ["false", "0", "no", "non", "नहीं", "לא", "nu", "ไม่ใช่", "否"],
+    # Add more entries as needed for other languages
+}
+
+
+
+maxm_METRICS = ["rouge_l", "cider", "exact_match", "relaxed_accuracy"]
+
+def maxm_doc_to_visual(doc):
+    # This is necessary, for reference check: https://huggingface.co/datasets/floschne/maxm 
+    pil_image = DatasetsImage().decode_example(doc["image"])
+    return [pil_image.convert("RGB")]
+
+def maxm_doc_to_text(doc, model_specific_prompt_kwargs=None):
+    question = doc["question"].strip()
+    if model_specific_prompt_kwargs:
+        pre_prompt = model_specific_prompt_kwargs.get("pre_prompt", "")
+        post_prompt = model_specific_prompt_kwargs.get("post_prompt", "")
+        question = f"{pre_prompt}{question}{post_prompt}"
+    return question
+
+def maxm_process_results(doc, result):
+    """
+    Args:
+        doc: a instance of the eval dataset
+        results: [pred]
+    Returns:
+        a dictionary with key: metric name, value: metric value
+    """
+    
+    pred = result[0] if len(result) > 0 else ""
+    image_id = doc["image_id"]
+    # convert str to int, replace a-z with 1-26
+    int_id = ""
+    for c in image_id:
+        if c.isalpha():
+            int_id += str(ord(c) - 96)
+        else:
+            int_id += c
+    
+    id = int(int_id)
+
+    data_dict = {"answer": doc["answers"], "pred": pred, "image_id": id}
+
+    return {f"{metric}": data_dict for metric in maxm_METRICS}
+
+def exact_match_with_multiple_references(predictions, references):
+    exact_matches = []
+    for pred, ref_list in zip(predictions, references):
+        if isinstance(pred, list):
+            pred = pred[-1]
+        match = pred in [ans for ans in ref_list] #any(pred == ref for ref in ref_list)
+        exact_matches.append(match)
+    return {"exact_match": 100*sum(exact_matches) / len(exact_matches)}
+
+def preprocess_answer(answer):
+    """Preprocess the answer by lowercasing, stripping, and removing punctuation."""
+    # Normalize unicode characters
+    answer = unicodedata.normalize('NFKD', answer)
+    # Lowercase (except for scripts where case doesn't apply, like Chinese)
+    answer = answer.lower()
+    # Strip whitespace
+    answer = answer.strip()
+    # Remove punctuation (keep language-specific characters intact)
+    answer = re.sub(r'[^\w\s]', '', answer)
+    return answer
+
+
+def is_correct(generated, gold_answers, synonyms=None):
+    """Check if the generated answer matches any of the gold answers using relaxed accuracy criteria."""
+    generated = preprocess_answer(generated)
+    
+    for gold in gold_answers:
+        gold = preprocess_answer(gold)
+
+        # Check for exact match
+        if generated == gold:
+            return True
+        
+        if generated in gold:
+            return True
+
+        # # Check if generated answer starts or ends with the gold answer
+        if generated.startswith(gold) or generated.endswith(gold):
+            return True
+
+        # Check for synonyms (e.g., yes/no, true/false)
+        if synonyms and generated in synonyms.get(gold, []):
+            return True
+
+    return False
+
+
+def generated_label_classification_evaluation(
+    gold_labels: List[str] | List[List[str]],
+    pred_labels: List[str],
+    vqa_post_process: bool = True,
+    bool_to_yes_no: bool = True,
+    entailment_to_yes_no_maybe: bool = False,
+    remove_trailing_period: bool = True,
+) -> Dict[str, float]:
+    single_answers = isinstance(gold_labels[0], str)
+    print(f"Evaluation with {single_answers=}")
+
+    df = pd.DataFrame(
+        {
+            "gold": gold_labels,
+            "pred": pred_labels,
+        }
+    )
+
+    if single_answers:
+        df["gold"] = df["gold"].apply(lambda x: str(x).strip())
+        if remove_trailing_period:
+            df["pred"] = df["pred"].apply(lambda x: str(x).rstrip("."))
+    else:
+        df["gold"] = df["gold"].apply(lambda x: [str(ans).strip() for ans in x])
+        if remove_trailing_period:
+            df["pred"] = df["pred"].apply(lambda x: [str(ans).rstrip(".") for ans in x])
+
+    if entailment_to_yes_no_maybe:
+
+        def _entailment_to_yes_no_maybe(x) -> Literal["yes", "no", "maybe"]:
+            if x.lower() in ["yes", "no", "maybe"]:
+                return x.lower()
+            elif x.lower() in ["true", "false"]:
+                return "yes" if x.lower() == "true" else "no"
+            elif x.lower() in ["1", "0"]:
+                return "yes" if x.lower() == "1" else "no"
+            elif x.lower() in ["entailment", "contradiction", "neutral"]:
+                if x.lower() == "entailment":
+                    return "yes"
+                elif x.lower() == "contradiction":
+                    return "no"
+                else:
+                    return "maybe"
+            else:
+                return x
+
+        df["pred"] = df["pred"].apply(_entailment_to_yes_no_maybe)
+        if single_answers:
+            df["gold"] = df["gold"].apply(_entailment_to_yes_no_maybe)
+        else:
+            df["gold"] = df["gold"].apply(
+                lambda x: [_entailment_to_yes_no_maybe(ans) for ans in x]
+            )
+
+    if bool_to_yes_no:
+
+        def map_bool_to_yes_no(x):
+            if isinstance(x, bool):
+                return "yes" if x else "no"
+            elif isinstance(x, str):
+                if x.lower() == "true":
+                    return "yes"
+                elif x.lower() == "false":
+                    return "no"
+            return x
+
+        df.pred = df.pred.apply(lambda x: str(map_bool_to_yes_no(x)).strip())
+        if single_answers:
+            df.gold = df.gold.apply(lambda ans: str(map_bool_to_yes_no(ans)).strip())
+        else:
+            df.gold = df.gold.apply(
+                lambda answers: [map_bool_to_yes_no(ans) for ans in answers]
+            )
+
+    df["pred_post_processed"] = vqa_clean(df["pred"].tolist())
+
+    # print(df.head())
+
+    scores: Dict[str, float] = {}
+    if single_answers:
+        scores["acc"] = (df["gold"] == df["pred"]).mean()
+        scores["relaxed_acc"] = df.apply(
+            lambda x: x["pred"].startswith(x["gold"]) or x["pred"].endswith(x["gold"]),
+            axis=1,
+        ).mean()
+
+        if vqa_post_process:
+            post_proc_acc = (df["gold"] == df["pred_post_processed"]).mean()
+            post_proc_relaxed_acc = df.apply(
+                lambda x: x["pred_post_processed"].startswith(x["gold"])
+                or x["pred_post_processed"].endswith(x["gold"]),
+                axis=1,
+            ).mean()
+            scores["acc_post_processed"] = post_proc_acc
+            scores["relaxed_acc_post_processed"] = post_proc_relaxed_acc
+    else:
+        scores["acc"] = df.apply(lambda x: x["pred"] in x["gold"], axis=1).mean()
+        scores["relaxed_acc"] = df.apply(
+            lambda x: any(
+                x["pred"].startswith(ans) or x["pred"].endswith(ans)
+                for ans in x["gold"]
+            ),
+            axis=1,
+        ).mean()
+
+        if vqa_post_process:
+            scores["acc_post_processed"] = df.apply(
+                lambda x: any(x["pred_post_processed"] == ans for ans in x["gold"]),
+                axis=1,
+            ).mean()
+            scores["relaxed_acc_post_processed"] = df.apply(
+                lambda x: any(
+                    x["pred_post_processed"].startswith(ans)
+                    or x["pred_post_processed"].endswith(ans)
+                    # or x["pred_post_processed"] in ans
+                    for ans in x["gold"]
+                ),
+                axis=1,
+            ).mean()
+            # print(df.head())
+
+    return scores
+
+
+
+# adapted from https://github.com/salesforce/LAVIS/blob/main/lavis/common/vqa_tools/vqa_eval.py
+def vqa_clean(labels: List[str]):
+    manualMap = {
+        "none": "0",
+        "zero": "0",
+        "one": "1",
+        "two": "2",
+        "three": "3",
+        "four": "4",
+        "five": "5",
+        "six": "6",
+        "seven": "7",
+        "eight": "8",
+        "nine": "9",
+        "ten": "10",
+    }
+    articles = ["a", "an", "the"]
+    periodStrip = re.compile(r"(?!<=\d)(\.)(?!\d)")
+    commaStrip = re.compile(r"(\d)(,)(\d)")
+    punct = [
+        ";",
+        r"/",
+        "[",
+        "]",
+        '"',
+        "{",
+        "}",
+        "(",
+        ")",
+        "=",
+        "+",
+        "\\",
+        "_",
+        "-",
+        ">",
+        "<",
+        "@",
+        "`",
+        ",",
+        "?",
+        "!",
+    ]
+    contractions = {
+        "aint": "ain't",
+        "arent": "aren't",
+        "cant": "can't",
+        "couldve": "could've",
+        "couldnt": "couldn't",
+        "couldn'tve": "couldn't've",
+        "couldnt've": "couldn't've",
+        "didnt": "didn't",
+        "doesnt": "doesn't",
+        "dont": "don't",
+        "hadnt": "hadn't",
+        "hadnt've": "hadn't've",
+        "hadn'tve": "hadn't've",
+        "hasnt": "hasn't",
+        "havent": "haven't",
+        "hed": "he'd",
+        "hed've": "he'd've",
+        "he'dve": "he'd've",
+        "hes": "he's",
+        "howd": "how'd",
+        "howll": "how'll",
+        "hows": "how's",
+        "Id've": "I'd've",
+        "I'dve": "I'd've",
+        "Im": "I'm",
+        "Ive": "I've",
+        "isnt": "isn't",
+        "itd": "it'd",
+        "itd've": "it'd've",
+        "it'dve": "it'd've",
+        "itll": "it'll",
+        "let's": "let's",
+        "maam": "ma'am",
+        "mightnt": "mightn't",
+        "mightnt've": "mightn't've",
+        "mightn'tve": "mightn't've",
+        "mightve": "might've",
+        "mustnt": "mustn't",
+        "mustve": "must've",
+        "neednt": "needn't",
+        "notve": "not've",
+        "oclock": "o'clock",
+        "oughtnt": "oughtn't",
+        "ow's'at": "'ow's'at",
+        "'ows'at": "'ow's'at",
+        "'ow'sat": "'ow's'at",
+        "shant": "shan't",
+        "shed've": "she'd've",
+        "she'dve": "she'd've",
+        "she's": "she's",
+        "shouldve": "should've",
+        "shouldnt": "shouldn't",
+        "shouldnt've": "shouldn't've",
+        "shouldn'tve": "shouldn't've",
+        "somebody'd": "somebodyd",
+        "somebodyd've": "somebody'd've",
+        "somebody'dve": "somebody'd've",
+        "somebodyll": "somebody'll",
+        "somebodys": "somebody's",
+        "someoned": "someone'd",
+        "someoned've": "someone'd've",
+        "someone'dve": "someone'd've",
+        "someonell": "someone'll",
+        "someones": "someone's",
+        "somethingd": "something'd",
+        "somethingd've": "something'd've",
+        "something'dve": "something'd've",
+        "somethingll": "something'll",
+        "thats": "that's",
+        "thered": "there'd",
+        "thered've": "there'd've",
+        "there'dve": "there'd've",
+        "therere": "there're",
+        "theres": "there's",
+        "theyd": "they'd",
+        "theyd've": "they'd've",
+        "they'dve": "they'd've",
+        "theyll": "they'll",
+        "theyre": "they're",
+        "theyve": "they've",
+        "twas": "'twas",
+        "wasnt": "wasn't",
+        "wed've": "we'd've",
+        "we'dve": "we'd've",
+        "weve": "we've",
+        "werent": "weren't",
+        "whatll": "what'll",
+        "whatre": "what're",
+        "whats": "what's",
+        "whatve": "what've",
+        "whens": "when's",
+        "whered": "where'd",
+        "wheres": "where's",
+        "whereve": "where've",
+        "whod": "who'd",
+        "whod've": "who'd've",
+        "who'dve": "who'd've",
+        "wholl": "who'll",
+        "whos": "who's",
+        "whove": "who've",
+        "whyll": "why'll",
+        "whyre": "why're",
+        "whys": "why's",
+        "wont": "won't",
+        "wouldve": "would've",
+        "wouldnt": "wouldn't",
+        "wouldnt've": "wouldn't've",
+        "wouldn'tve": "wouldn't've",
+        "yall": "y'all",
+        "yall'll": "y'all'll",
+        "y'allll": "y'all'll",
+        "yall'd've": "y'all'd've",
+        "y'alld've": "y'all'd've",
+        "y'all'dve": "y'all'd've",
+        "youd": "you'd",
+        "youd've": "you'd've",
+        "you'dve": "you'd've",
+        "youll": "you'll",
+        "youre": "you're",
+        "youve": "you've",
+    }
+
+    def processPunctuation(inText):
+        outText = inText
+        for p in punct:
+            if (p + " " in inText or " " + p in inText) or (
+                re.search(commaStrip, inText) is not None
+            ):
+                outText = outText.replace(p, "")
+            else:
+                outText = outText.replace(p, " ")
+        outText = periodStrip.sub("", outText, re.UNICODE)
+        return outText
+
+    def processDigitArticle(inText):
+        outText = []
+        tempText = inText.lower().split()
+        for word in tempText:
+            word = manualMap.setdefault(word, word)
+            if word not in articles:
+                outText.append(word)
+            else:
+                pass
+        for wordId, word in enumerate(outText):
+            if word in contractions:
+                outText[wordId] = contractions[word]
+        outText = " ".join(outText)
+        return outText
+
+    cleaned_labels = []
+    for label in labels:
+        label = label.replace("\n", "").replace("\t", "").strip()
+        label = processPunctuation(label)
+        label = processDigitArticle(label)
+        label = ''.join(label)
+        label = label.strip("'")
+        cleaned_labels.append(label)
+    # print(cleaned_labels)
+    return cleaned_labels
+
+
+def relaxed_accuracy_metric(generated_answers, gold_answers, synonyms=None):
+    """Calculate the relaxed accuracy for a list of generated and gold answers."""
+    correct = 0
+    for gen_ans, gold_ans in zip(generated_answers, gold_answers):
+        if is_correct(gen_ans, gold_ans, synonyms=synonyms):
+            correct += 1
+    return 100*correct / len(generated_answers)
+
+
+def maxm_aggregate_results_v2(results, metric, args):
+
+    print('Currently in Metric: ', metric)
+
+    if metric == "exact_match":
+
+        preds = []
+        for r in results:
+            if isinstance(r["pred"], list) and len(r["pred"]) > 1:
+                preds.append(r["pred"][-1])
+            else:
+                preds.append(r["pred"])
+            # r["pred"][-1] for r in results if len(r["pred"])>1 else r["pred"]]
+        res = [r["answer"] for r in results]
+
+        return exact_match_with_multiple_references(preds, res)['exact_match']
+    
+    if metric == 'relaxed_accuracy':
+        preds = []
+        for r in results:
+            if isinstance(r["pred"], list) and len(r["pred"]) > 1:
+                preds.append([r["pred"][-1]])
+            else:
+                preds.append([r["pred"]])
+            # r["pred"][-1] for r in results if len(r["pred"])>1 else r["pred"]]
+        res = [r["answer"] for r in results]
+        # print('Preds: ', preds[:5])
+        # print('Res: ', res[:5])
+        # return relaxed_accuracy_metric(preds, res, synonyms)
+        # return relaxed_accuracy_metric(preds, res)
+        return 100*generated_label_classification_evaluation(res, preds)['relaxed_acc_post_processed']
+
+
+    scorers = [(Rouge(), "rouge_l"), (Cider(), "cider")]
+    scorers_dict = {s[1]: s for s in scorers}
+
+    # Prepare ground truth and prediction data for COCO format
+    ground_truths = []
+    predictions = []
+    idx = 0
+    for item in results:
+        image_id = item['image_id']
+        for answer in item['answer']:
+            ground_truths.append({
+                'image_id': image_id,
+                'caption': answer,
+                'id': idx
+            })
+            idx += 1
+        predictions.append({
+            'image_id': image_id,
+            'caption': item['pred']
+        })
+
+    # Create COCO-like annotations for ground truth
+    coco = COCO()
+    coco.dataset = {'images': [{'id': item['image_id']} for item in results], 'annotations': ground_truths}
+    coco.createIndex()
+
+    # Create a fake results file
+    coco_result = coco.loadRes(predictions)
+
+    # Evaluation setup
+    coco_eval = COCOEvalCap(coco, coco_result)
+    imgIds = coco_eval.params["image_id"]
+    gts = {}
+    res = {}
+    for imgId in imgIds:
+        gts[imgId] = coco_eval.coco.imgToAnns[imgId]
+        res[imgId] = coco_eval.cocoRes.imgToAnns[imgId]
+
+
+    for k,v in res.items():
+        if len(v)>1:
+            res[k] = [v[-1]]
+
+    # Tokenization
+    eval_logger.info("tokenization...")
+    tokenizer = PTBTokenizer()
+    gts = tokenizer.tokenize(gts)
+    res = tokenizer.tokenize(res)
+
+    eval_logger.info(f"Computing {metric} scores...")
+
+    score, scores = scorers_dict[metric][0].compute_score(gts, res)
+
+    path = generate_submission_file(f"maxm_results_{metric}.json", args)
+    if not os.path.exists(path):
+        eval_logger.info("Storing prediction that can be submitted to the server ...")
+        with open(path, "w") as f:
+            json.dump(predictions, f, indent=4)
+
+    return 100*score
+
+
+def maxm_ema(results, args):
+    return maxm_aggregate_results_v2(results, "exact_match", args)
+
+
+def maxm_rouge_l(results, args):
+    # print('Rouge input: ', results)
+    return maxm_aggregate_results_v2(results, "rouge_l", args)
+
+def maxm_cider(results, args):
+    return maxm_aggregate_results_v2(results, "cider", args)
+
+
+def maxm_relaxed_ema(results, args):
+    return maxm_aggregate_results_v2(results, "relaxed_accuracy", args)
\ No newline at end of file
diff --git a/lmms_eval/tasks/xgqa/utils.py b/lmms_eval/tasks/xgqa/utils.py
new file mode 100755
index 000000000..71ff2cd4d
--- /dev/null
+++ b/lmms_eval/tasks/xgqa/utils.py
@@ -0,0 +1,18 @@
+
+def xgqa_doc_to_visual(doc):
+    return [doc['image'].convert("RGB")]
+
+
+def xgqa_doc_to_text(doc, model_specific_prompt_kwargs):
+    question = doc["question"].strip()
+    pre_prompt = model_specific_prompt_kwargs["pre_prompt"]
+    post_prompt = model_specific_prompt_kwargs["post_prompt"]
+    return f"{pre_prompt}{question} {post_prompt.strip()}"
+
+def xgqa_process_result(doc, results):
+    target = doc['answer'].strip().lower()
+    pred = results[0]
+    pred = pred.strip().lower()
+    if target in pred:
+        return {"exact_match": 1.0}
+    return {"exact_match": 0.0}
\ No newline at end of file
diff --git a/lmms_eval/tasks/xgqa/xgqa.yaml b/lmms_eval/tasks/xgqa/xgqa.yaml
new file mode 100755
index 000000000..f2b928e71
--- /dev/null
+++ b/lmms_eval/tasks/xgqa/xgqa.yaml
@@ -0,0 +1,10 @@
+group: xgqa
+task:
+  - xgqa_bn
+  - xgqa_de
+  - xgqa_en
+  - xgqa_id
+  - xgqa_ko
+  - xgqa_pt
+  - xgqa_ru
+  - xgqa_zh
\ No newline at end of file
diff --git a/lmms_eval/tasks/xgqa/xgqa_bn.yaml b/lmms_eval/tasks/xgqa/xgqa_bn.yaml
new file mode 100755
index 000000000..40932064b
--- /dev/null
+++ b/lmms_eval/tasks/xgqa/xgqa_bn.yaml
@@ -0,0 +1,33 @@
+dataset_path: neulab/PangeaBench-xgqa
+dataset_name: default
+dataset_kwargs:
+  token: True
+task: "xgqa_bn"
+test_split: bn
+output_type: generate_until
+doc_to_visual: !function utils.xgqa_doc_to_visual
+doc_to_text: !function utils.xgqa_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+process_results: !function utils.xgqa_process_result
+metadata:
+  - version: 0.0
+  
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer the question using a single word or phrase in English."
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/xgqa/xgqa_de.yaml b/lmms_eval/tasks/xgqa/xgqa_de.yaml
new file mode 100755
index 000000000..2b11b3ddc
--- /dev/null
+++ b/lmms_eval/tasks/xgqa/xgqa_de.yaml
@@ -0,0 +1,33 @@
+dataset_path: neulab/PangeaBench-xgqa
+dataset_name: default
+dataset_kwargs:
+  token: True
+task: "xgqa_de"
+test_split: de
+output_type: generate_until
+doc_to_visual: !function utils.xgqa_doc_to_visual
+doc_to_text: !function utils.xgqa_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+process_results: !function utils.xgqa_process_result
+metadata:
+  - version: 0.0
+  
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer the question using a single word or phrase in English."
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/xgqa/xgqa_en.yaml b/lmms_eval/tasks/xgqa/xgqa_en.yaml
new file mode 100755
index 000000000..3f5b3aff1
--- /dev/null
+++ b/lmms_eval/tasks/xgqa/xgqa_en.yaml
@@ -0,0 +1,33 @@
+dataset_path: neulab/PangeaBench-xgqa
+dataset_name: default
+dataset_kwargs:
+  token: True
+task: "xgqa_en"
+test_split: en
+output_type: generate_until
+doc_to_visual: !function utils.xgqa_doc_to_visual
+doc_to_text: !function utils.xgqa_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+process_results: !function utils.xgqa_process_result
+metadata:
+  - version: 0.0
+  
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer the question using a single word or phrase."
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/xgqa/xgqa_id.yaml b/lmms_eval/tasks/xgqa/xgqa_id.yaml
new file mode 100755
index 000000000..171cf442c
--- /dev/null
+++ b/lmms_eval/tasks/xgqa/xgqa_id.yaml
@@ -0,0 +1,33 @@
+dataset_path: neulab/PangeaBench-xgqa
+dataset_name: default
+dataset_kwargs:
+  token: True
+task: "xgqa_id"
+test_split: id
+output_type: generate_until
+doc_to_visual: !function utils.xgqa_doc_to_visual
+doc_to_text: !function utils.xgqa_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+process_results: !function utils.xgqa_process_result
+metadata:
+  - version: 0.0
+  
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer the question using a single word or phrase in English."
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/xgqa/xgqa_ko.yaml b/lmms_eval/tasks/xgqa/xgqa_ko.yaml
new file mode 100755
index 000000000..dc6ad0a4d
--- /dev/null
+++ b/lmms_eval/tasks/xgqa/xgqa_ko.yaml
@@ -0,0 +1,33 @@
+dataset_path: neulab/PangeaBench-xgqa
+dataset_name: default
+dataset_kwargs:
+  token: True
+task: "xgqa_ko"
+test_split: ko
+output_type: generate_until
+doc_to_visual: !function utils.xgqa_doc_to_visual
+doc_to_text: !function utils.xgqa_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+process_results: !function utils.xgqa_process_result
+metadata:
+  - version: 0.0
+  
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer the question using a single word or phrase in English."
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/xgqa/xgqa_pt.yaml b/lmms_eval/tasks/xgqa/xgqa_pt.yaml
new file mode 100755
index 000000000..4cf96ce10
--- /dev/null
+++ b/lmms_eval/tasks/xgqa/xgqa_pt.yaml
@@ -0,0 +1,33 @@
+dataset_path: neulab/PangeaBench-xgqa
+dataset_name: default
+dataset_kwargs:
+  token: True
+task: "xgqa_pt"
+test_split: pt
+output_type: generate_until
+doc_to_visual: !function utils.xgqa_doc_to_visual
+doc_to_text: !function utils.xgqa_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+process_results: !function utils.xgqa_process_result
+metadata:
+  - version: 0.0
+  
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer the question using a single word or phrase in English."
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/xgqa/xgqa_ru.yaml b/lmms_eval/tasks/xgqa/xgqa_ru.yaml
new file mode 100755
index 000000000..d2d08dfa1
--- /dev/null
+++ b/lmms_eval/tasks/xgqa/xgqa_ru.yaml
@@ -0,0 +1,33 @@
+dataset_path: neulab/PangeaBench-xgqa
+dataset_name: default
+dataset_kwargs:
+  token: True
+task: "xgqa_ru"
+test_split: ru
+output_type: generate_until
+doc_to_visual: !function utils.xgqa_doc_to_visual
+doc_to_text: !function utils.xgqa_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+process_results: !function utils.xgqa_process_result
+metadata:
+  - version: 0.0
+  
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer the question using a single word or phrase in English."
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/xgqa/xgqa_zh.yaml b/lmms_eval/tasks/xgqa/xgqa_zh.yaml
new file mode 100755
index 000000000..281b79516
--- /dev/null
+++ b/lmms_eval/tasks/xgqa/xgqa_zh.yaml
@@ -0,0 +1,33 @@
+dataset_path: neulab/PangeaBench-xgqa
+dataset_name: default
+dataset_kwargs:
+  token: True
+task: "xgqa_zh"
+test_split: zh
+output_type: generate_until
+doc_to_visual: !function utils.xgqa_doc_to_visual
+doc_to_text: !function utils.xgqa_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+process_results: !function utils.xgqa_process_result
+metadata:
+  - version: 0.0
+  
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer the question using a single word or phrase in English."
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/utils.py b/lmms_eval/tasks/xm100/utils.py
new file mode 100644
index 000000000..e22f10a9d
--- /dev/null
+++ b/lmms_eval/tasks/xm100/utils.py
@@ -0,0 +1,202 @@
+import os
+import json
+from pycocoevalcap.eval import COCOEvalCap, Bleu, Meteor, Rouge, Cider, Spice
+from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer
+from pycocotools.coco import COCO
+import unicodedata
+
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+
+import logging
+
+eval_logger = logging.getLogger("lmms-eval")
+
+dir_name = os.path.dirname(os.path.abspath(__file__))
+
+xm100_METRICS = ["Bleu_4", "Bleu_3", "Bleu_2", "Bleu_1", "METEOR", "ROUGE_L", "CIDEr"]  # , "SPICE"]
+
+
+def xm100_doc_to_visual(doc):
+    return [doc["image"].convert("RGB")]
+
+
+def xm100_doc_to_text(doc):
+    return f"Provide a one-sentence caption for the provided image."
+
+
+def xm100_process_result(doc, result):
+    """
+    Args:
+        doc: a instance of the eval dataset
+        results: [pred]
+    Returns:
+        a dictionary with key: metric name, value: metric value
+    """
+    pred = result[0] if len(result) > 0 else ""
+    # question_id = doc["question_id"]
+    # The question id in our dataset is the image file itself
+    # image_id = int(question_id.split("_")[-1].split(".")[0])
+    # id = doc["id"]
+    image_id = doc["image_id"]
+    # convert str to int, replace a-z with 1-26
+    int_id = ""
+    for c in image_id:
+        if c.isalpha():
+            int_id += str(ord(c) - 96)
+        else:
+            int_id += c
+    
+    id = int(int_id)
+
+    
+
+    data_dict = {"answer": [doc["caption"]], "pred": pred, "image_id": id}
+
+    return {f"xm100_{metric}": data_dict for metric in xm100_METRICS}
+
+
+def normalize(result, args):
+    caption = result["pred"]
+    if 'xm100_zh' in args.tasks:
+        from spacy.lang.zh import Chinese
+        chinese = Chinese() #.from_config({"nlp": {"tokenizer": {"segmenter": "jieba"}}})
+        caption = "".join([word.text for word in chinese(caption)])
+    if 'xm100_jp' in args.tasks:
+        from spacy.lang.ja import Japanese
+        japanese = Japanese()
+        caption = "".join([word.text for word in japanese(caption)])
+    if 'xm100_th' in args.tasks:
+        from spacy.lang.th import Thai
+        thai = Thai()
+        caption = "".join([word.text for word in thai(caption)])
+    caption = unicodedata.normalize("NFC", caption)
+    result["pred"] = caption
+    return result
+
+def xm100_aggregation_result(results, metric, args):
+    scorers = [(Bleu(4), "Bleu_1"), (Bleu(4), "Bleu_2"), (Bleu(4), "Bleu_3"), (Bleu(4), "Bleu_4"), (Meteor(), "METEOR"), (Rouge(), "ROUGE_L"), (Cider(), "CIDEr"), (Spice(), "SPICE")]
+    scorers_dict = {s[1]: s for s in scorers}
+
+    stored_results = []
+    # In order to make the coco eval tools to successfully create index
+    # We need at least two dict in the dataset
+    # 'annotation' and 'images'
+    # 'annotation' exactly reproduce the original annotation
+    # 'images' however only need the image id which is contained in the file name
+    dataset = {"annotations": [], "images": []}
+    idx = 0
+    for result in results:
+        result = normalize(result, args)
+        stored_results.append({"image_id": int(result["image_id"]), "caption": result["pred"]})
+        for a in result["answer"]:
+            dataset["annotations"].append({"image_id": int(result["image_id"]), "caption": a, "id": idx})
+            idx += 1
+        dataset["images"].append({"id": result["image_id"]})
+
+    coco = COCO()
+    # Manually create index here
+    coco.dataset = dataset
+    coco.createIndex()
+
+    coco_result = coco.loadRes(stored_results)
+    coco_eval = COCOEvalCap(coco, coco_result)
+
+    imgIds = coco_eval.params["image_id"]
+    gts = {}
+    res = {}
+    for imgId in imgIds:
+        gts[imgId] = coco_eval.coco.imgToAnns[imgId]
+        res[imgId] = coco_eval.cocoRes.imgToAnns[imgId]
+
+    eval_logger.info("tokenization...")
+
+    # if 'xm100_zh' not in args.tasks and 'xm100_jp' not in args.tasks and 'xm100_th' not in args.tasks:
+    tokenizer = PTBTokenizer()
+    gts = tokenizer.tokenize(gts)
+    res = tokenizer.tokenize(res)
+
+    eval_logger.info(f"Computing {metric} scores...")
+
+    score, scores = scorers_dict[metric][0].compute_score(gts, res)
+    # When metric is one of the Bleu, score will be a list
+    if type(score) == list:
+        n = int(metric.split("_")[-1])
+        score = score[n - 1]
+
+    path = generate_submission_file("xm100_captions_val_alg_results.json", args)
+    if not os.path.exists(path):
+        eval_logger.info("Storing prediction that can be submitted to the server ...")
+        with open(path, "w") as f:
+            json.dump(stored_results, f, indent=4)
+
+    return score
+
+
+def xm100_bleu4(results, args):
+    return xm100_aggregation_result(results, "Bleu_4", args)
+
+
+def xm100_bleu3(results, args):
+    return xm100_aggregation_result(results, "Bleu_3", args)
+
+
+def xm100_bleu2(results, args):
+    return xm100_aggregation_result(results, "Bleu_2", args)
+
+
+def xm100_bleu1(results, args):
+    return xm100_aggregation_result(results, "Bleu_1", args)
+
+
+def xm100_meteor(results, args):
+    return xm100_aggregation_result(results, "METEOR", args)
+
+
+def xm100_rougel(results, args):
+    return xm100_aggregation_result(results, "ROUGE_L", args)
+
+
+def xm100_cider(results, args):
+    return xm100_aggregation_result(results, "CIDEr", args)
+
+
+def xm100_spice(results, args):
+    return xm100_aggregation_result(results, "SPICE", args)
+
+
+def xm100_test_process_result(doc, result):
+    """
+    Args:
+        doc: a instance of the eval dataset
+        results: [pred]
+    Returns:
+        a dictionary with key: metric name (in this case coco_passthrough), value: metric value
+    """
+    # question_id = doc["question_id"]
+    # # The question id in our dataset is the image file itself
+    # image_id = int(question_id.split("_")[-1].split(".")[0])
+    image_id = doc["image_id"]
+    # convert str to int, replace a-z with 1-26
+    int_id = ""
+    for c in image_id:
+        if c.isalpha():
+            int_id += str(ord(c) - 96)
+        else:
+            int_id += c
+    
+    id = int(int_id)
+    return {"xm100_passthrough": {"pred": result, "image_id": id}}
+
+
+def xm100_test_aggregation_result(results, args):
+    stored_results = []
+    for result in results:
+        result = normalize(result, args)
+        stored_results.append({"image_id": int(result["image_id"]), "caption": result["pred"]})
+
+    path = generate_submission_file("xm100_captions_test_alg_results.json", args)
+    eval_logger.info("Storing prediction that can be submitted to the server ...")
+    with open(path, "w") as f:
+        json.dump(stored_results, f, indent=4)
+
+    eval_logger.info(f"Your test result has been stored in to {path}. Make sure you also have the val result stored to submit to the server on https://codalab.lisn.upsaclay.fr/competitions/7404#participate.")
diff --git a/lmms_eval/tasks/xm100/xm100.yaml b/lmms_eval/tasks/xm100/xm100.yaml
new file mode 100644
index 000000000..4263ce18d
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100.yaml
@@ -0,0 +1,38 @@
+group : xm100
+task:
+  - xm100_ar
+  - xm100_bn
+  - xm100_cs
+  - xm100_da
+  - xm100_de
+  - xm100_el
+  - xm100_en
+  - xm100_es
+  - xm100_fa
+  - xm100_fi
+  - xm100_fil
+  - xm100_fr
+  - xm100_hi
+  - xm100_hr
+  - xm100_hu
+  - xm100_id
+  - xm100_it
+  - xm100_he
+  - xm100_ja
+  - xm100_ko
+  - xm100_mi
+  - xm100_nl
+  - xm100_no
+  - xm100_pl
+  - xm100_pt
+  - xm100_quz
+  - xm100_ro
+  - xm100_ru
+  - xm100_sv
+  - xm100_sw
+  - xm100_te
+  - xm100_th
+  - xm100_tr
+  - xm100_uk
+  - xm100_vi
+  - xm100_zh
diff --git a/lmms_eval/tasks/xm100/xm100_ar.yaml b/lmms_eval/tasks/xm100/xm100_ar.yaml
new file mode 100644
index 000000000..560de4e0e
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_ar.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_ar"
+group : "xm100"
+test_split: ar
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Arabic."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_bn.yaml b/lmms_eval/tasks/xm100/xm100_bn.yaml
new file mode 100644
index 000000000..901bce18b
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_bn.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_bn"
+group : "xm100"
+test_split: bn
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Bengali."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_cs.yaml b/lmms_eval/tasks/xm100/xm100_cs.yaml
new file mode 100644
index 000000000..e6481e9d0
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_cs.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_cs"
+group : "xm100"
+test_split: cs
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Czech."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_da.yaml b/lmms_eval/tasks/xm100/xm100_da.yaml
new file mode 100644
index 000000000..af9840fcd
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_da.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_da"
+group : "xm100"
+test_split: da
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Danish."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_de.yaml b/lmms_eval/tasks/xm100/xm100_de.yaml
new file mode 100644
index 000000000..405072bde
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_de.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_de"
+group : "xm100"
+test_split: de
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in German."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_el.yaml b/lmms_eval/tasks/xm100/xm100_el.yaml
new file mode 100644
index 000000000..d023009b0
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_el.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_el"
+group : "xm100"
+test_split: el
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Greek."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_en.yaml b/lmms_eval/tasks/xm100/xm100_en.yaml
new file mode 100644
index 000000000..8fb63bb86
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_en.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_en"
+group : "xm100"
+test_split: en
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_es.yaml b/lmms_eval/tasks/xm100/xm100_es.yaml
new file mode 100644
index 000000000..5a0adb7cb
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_es.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_es"
+group : "xm100"
+test_split: es
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Spanish."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_fa.yaml b/lmms_eval/tasks/xm100/xm100_fa.yaml
new file mode 100644
index 000000000..3e94934d8
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_fa.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_fa"
+group : "xm100"
+test_split: fa
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Persian."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_fi.yaml b/lmms_eval/tasks/xm100/xm100_fi.yaml
new file mode 100644
index 000000000..6d6c379f8
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_fi.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_fi"
+group : "xm100"
+test_split: fi
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Finnish."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_fil.yaml b/lmms_eval/tasks/xm100/xm100_fil.yaml
new file mode 100644
index 000000000..e2e6d4c71
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_fil.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_fil"
+group : "xm100"
+test_split: fil
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Filipino."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_fr.yaml b/lmms_eval/tasks/xm100/xm100_fr.yaml
new file mode 100644
index 000000000..77c20f5c6
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_fr.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_fr"
+group : "xm100"
+test_split: fr
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in French."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_he.yaml b/lmms_eval/tasks/xm100/xm100_he.yaml
new file mode 100644
index 000000000..1ae30012d
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_he.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_he"
+group : "xm100"
+test_split: he
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Hebrew."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_hi.yaml b/lmms_eval/tasks/xm100/xm100_hi.yaml
new file mode 100644
index 000000000..62e5fe67a
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_hi.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_hi"
+group : "xm100"
+test_split: hi
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Hindi."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_hr.yaml b/lmms_eval/tasks/xm100/xm100_hr.yaml
new file mode 100644
index 000000000..742f91e1a
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_hr.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_hr"
+group : "xm100"
+test_split: hr
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Croatian."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_hu.yaml b/lmms_eval/tasks/xm100/xm100_hu.yaml
new file mode 100644
index 000000000..1fecde9f0
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_hu.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_hu"
+group : "xm100"
+test_split: hu
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Hungarian."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_id.yaml b/lmms_eval/tasks/xm100/xm100_id.yaml
new file mode 100644
index 000000000..52bef37ce
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_id.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_id"
+group : "xm100"
+test_split: id
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Indonesian."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_it.yaml b/lmms_eval/tasks/xm100/xm100_it.yaml
new file mode 100644
index 000000000..fd95beda2
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_it.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_it"
+group : "xm100"
+test_split: it
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Italian."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_ja.yaml b/lmms_eval/tasks/xm100/xm100_ja.yaml
new file mode 100644
index 000000000..f23e26d35
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_ja.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_ja"
+group : "xm100"
+test_split: ja
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Japanese."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_ko.yaml b/lmms_eval/tasks/xm100/xm100_ko.yaml
new file mode 100644
index 000000000..35fd7ef92
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_ko.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_ko"
+group : "xm100"
+test_split: ko
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Korean."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_mi.yaml b/lmms_eval/tasks/xm100/xm100_mi.yaml
new file mode 100644
index 000000000..cb900e359
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_mi.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_mi"
+group : "xm100"
+test_split: mi
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Maori."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_nl.yaml b/lmms_eval/tasks/xm100/xm100_nl.yaml
new file mode 100644
index 000000000..d324e0045
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_nl.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_nl"
+group : "xm100"
+test_split: nl
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Dutch."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_no.yaml b/lmms_eval/tasks/xm100/xm100_no.yaml
new file mode 100644
index 000000000..a4a7b7d71
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_no.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_no"
+group : "xm100"
+test_split: 'no'
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Norwegian."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_pl.yaml b/lmms_eval/tasks/xm100/xm100_pl.yaml
new file mode 100644
index 000000000..70a4dfeb5
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_pl.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_pl"
+group : "xm100"
+test_split: pl
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Polish."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_pt.yaml b/lmms_eval/tasks/xm100/xm100_pt.yaml
new file mode 100644
index 000000000..cee984498
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_pt.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_pt"
+group : "xm100"
+test_split: pt
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Portuguese."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_quz.yaml b/lmms_eval/tasks/xm100/xm100_quz.yaml
new file mode 100644
index 000000000..c76b7a773
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_quz.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_quz"
+group : "xm100"
+test_split: quz
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Quechua."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_ro.yaml b/lmms_eval/tasks/xm100/xm100_ro.yaml
new file mode 100644
index 000000000..cb70a45b3
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_ro.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_ro"
+group : "xm100"
+test_split: ro
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Romanian."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_ru.yaml b/lmms_eval/tasks/xm100/xm100_ru.yaml
new file mode 100644
index 000000000..f837f7297
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_ru.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_ru"
+group : "xm100"
+test_split: ru
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Russian."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_sv.yaml b/lmms_eval/tasks/xm100/xm100_sv.yaml
new file mode 100644
index 000000000..f78a831c2
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_sv.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_sv"
+group : "xm100"
+test_split: sv
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Swedish."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_sw.yaml b/lmms_eval/tasks/xm100/xm100_sw.yaml
new file mode 100644
index 000000000..d751b71fe
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_sw.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_sw"
+group : "xm100"
+test_split: sw
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Swahili."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_te.yaml b/lmms_eval/tasks/xm100/xm100_te.yaml
new file mode 100644
index 000000000..0584da374
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_te.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_te"
+group : "xm100"
+test_split: te
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Telugu."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_th.yaml b/lmms_eval/tasks/xm100/xm100_th.yaml
new file mode 100644
index 000000000..e9ef7aa27
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_th.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_th"
+group : "xm100"
+test_split: th
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Thai."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_tr.yaml b/lmms_eval/tasks/xm100/xm100_tr.yaml
new file mode 100644
index 000000000..528c50f95
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_tr.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_tr"
+group : "xm100"
+test_split: tr
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Turkish."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_uk.yaml b/lmms_eval/tasks/xm100/xm100_uk.yaml
new file mode 100644
index 000000000..bcf1b1b78
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_uk.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_uk"
+group : "xm100"
+test_split: uk
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Ukrainian."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
diff --git a/lmms_eval/tasks/xm100/xm100_vi.yaml b/lmms_eval/tasks/xm100/xm100_vi.yaml
new file mode 100644
index 000000000..8128e0100
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_vi.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_vi"
+group : "xm100"
+test_split: vi
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Vietnamese."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xm100/xm100_zh.yaml b/lmms_eval/tasks/xm100/xm100_zh.yaml
new file mode 100644
index 000000000..5ab7ae312
--- /dev/null
+++ b/lmms_eval/tasks/xm100/xm100_zh.yaml
@@ -0,0 +1,42 @@
+dataset_path: neulab/PangeaBench-xm100
+dataset_kwargs:
+  token: True
+task : "xm100_zh"
+group : "xm100"
+test_split: zh
+output_type: generate_until
+doc_to_visual: !function utils.xm100_doc_to_visual
+doc_to_text: "Provide a one-sentence caption for the provided image in Chinese."
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.xm100_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: xm100_Bleu_4 
+    aggregation : !function utils.xm100_bleu4
+    higher_is_better : true
+  - metric: xm100_Bleu_3
+    aggregation : !function utils.xm100_bleu3
+    higher_is_better : true
+  - metric: xm100_Bleu_2
+    aggregation : !function utils.xm100_bleu2
+    higher_is_better : true
+  - metric: xm100_Bleu_1
+    aggregation : !function utils.xm100_bleu1
+    higher_is_better : true
+  - metric: xm100_METEOR
+    aggregation : !function utils.xm100_meteor
+    higher_is_better : true
+  - metric: xm100_ROUGE_L
+    aggregation : !function utils.xm100_rougel
+    higher_is_better : true
+  - metric: xm100_CIDEr
+    aggregation : !function utils.xm100_cider
+    higher_is_better : true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xmmmu/mmmu_Arabic_val.yaml b/lmms_eval/tasks/xmmmu/mmmu_Arabic_val.yaml
new file mode 100644
index 000000000..23990de6a
--- /dev/null
+++ b/lmms_eval/tasks/xmmmu/mmmu_Arabic_val.yaml
@@ -0,0 +1,21 @@
+dataset_path: neulab/PangeaBench-xmmmu
+task: "mmmu_Arabic_val"
+test_split: ar
+output_type: generate_until
+doc_to_visual: !function utils.mmmu_doc_to_visual
+doc_to_text: !function utils.mmmu_doc_to_text
+doc_to_target: "answer"
+# The return value of process_results will be used by metrics
+process_results: !function utils.mmmu_process_results
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+generation_kwargs:
+  max_new_tokens: 16
+model_specific_generation_kwargs:
+  llava:
+    image_aspect_ratio: original
+metric_list:
+  - metric: mmmu_acc
+    aggregation: !function utils.mmmu_aggregate_results
+    higher_is_better: true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xmmmu/mmmu_English_val.yaml b/lmms_eval/tasks/xmmmu/mmmu_English_val.yaml
new file mode 100644
index 000000000..a87b3cc6d
--- /dev/null
+++ b/lmms_eval/tasks/xmmmu/mmmu_English_val.yaml
@@ -0,0 +1,21 @@
+dataset_path: neulab/PangeaBench-xmmmu
+task: "mmmu_English_val"
+test_split: en
+output_type: generate_until
+doc_to_visual: !function utils.mmmu_doc_to_visual
+doc_to_text: !function utils.mmmu_doc_to_text
+doc_to_target: "answer"
+# The return value of process_results will be used by metrics
+process_results: !function utils.mmmu_process_results
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+generation_kwargs:
+  max_new_tokens: 16
+model_specific_generation_kwargs:
+  llava:
+    image_aspect_ratio: original
+metric_list:
+  - metric: mmmu_acc
+    aggregation: !function utils.mmmu_aggregate_results
+    higher_is_better: true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xmmmu/mmmu_French_val.yaml b/lmms_eval/tasks/xmmmu/mmmu_French_val.yaml
new file mode 100644
index 000000000..ad19bcbb8
--- /dev/null
+++ b/lmms_eval/tasks/xmmmu/mmmu_French_val.yaml
@@ -0,0 +1,21 @@
+dataset_path: neulab/PangeaBench-xmmmu
+task: "mmmu_French_val"
+test_split: fr
+output_type: generate_until
+doc_to_visual: !function utils.mmmu_doc_to_visual
+doc_to_text: !function utils.mmmu_doc_to_text
+doc_to_target: "answer"
+# The return value of process_results will be used by metrics
+process_results: !function utils.mmmu_process_results
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+generation_kwargs:
+  max_new_tokens: 16
+model_specific_generation_kwargs:
+  llava:
+    image_aspect_ratio: original
+metric_list:
+  - metric: mmmu_acc
+    aggregation: !function utils.mmmu_aggregate_results
+    higher_is_better: true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xmmmu/mmmu_Hindi_val.yaml b/lmms_eval/tasks/xmmmu/mmmu_Hindi_val.yaml
new file mode 100644
index 000000000..c2aaf0bff
--- /dev/null
+++ b/lmms_eval/tasks/xmmmu/mmmu_Hindi_val.yaml
@@ -0,0 +1,21 @@
+dataset_path: neulab/PangeaBench-xmmmu
+task: "mmmu_Hindi_val"
+test_split: hi
+output_type: generate_until
+doc_to_visual: !function utils.mmmu_doc_to_visual
+doc_to_text: !function utils.mmmu_doc_to_text
+doc_to_target: "answer"
+# The return value of process_results will be used by metrics
+process_results: !function utils.mmmu_process_results
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+generation_kwargs:
+  max_new_tokens: 16
+model_specific_generation_kwargs:
+  llava:
+    image_aspect_ratio: original
+metric_list:
+  - metric: mmmu_acc
+    aggregation: !function utils.mmmu_aggregate_results
+    higher_is_better: true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xmmmu/mmmu_Indonesian_val.yaml b/lmms_eval/tasks/xmmmu/mmmu_Indonesian_val.yaml
new file mode 100644
index 000000000..503729f50
--- /dev/null
+++ b/lmms_eval/tasks/xmmmu/mmmu_Indonesian_val.yaml
@@ -0,0 +1,21 @@
+dataset_path: neulab/PangeaBench-xmmmu
+task: "mmmu_Indonesian_val"
+test_split: id
+output_type: generate_until
+doc_to_visual: !function utils.mmmu_doc_to_visual
+doc_to_text: !function utils.mmmu_doc_to_text
+doc_to_target: "answer"
+# The return value of process_results will be used by metrics
+process_results: !function utils.mmmu_process_results
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+generation_kwargs:
+  max_new_tokens: 16
+model_specific_generation_kwargs:
+  llava:
+    image_aspect_ratio: original
+metric_list:
+  - metric: mmmu_acc
+    aggregation: !function utils.mmmu_aggregate_results
+    higher_is_better: true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xmmmu/mmmu_Japanese_val.yaml b/lmms_eval/tasks/xmmmu/mmmu_Japanese_val.yaml
new file mode 100644
index 000000000..0c2f71f2d
--- /dev/null
+++ b/lmms_eval/tasks/xmmmu/mmmu_Japanese_val.yaml
@@ -0,0 +1,21 @@
+dataset_path: neulab/PangeaBench-xmmmu
+task: "mmmu_Japanese_val"
+test_split: ja
+output_type: generate_until
+doc_to_visual: !function utils.mmmu_doc_to_visual
+doc_to_text: !function utils.mmmu_doc_to_text
+doc_to_target: "answer"
+# The return value of process_results will be used by metrics
+process_results: !function utils.mmmu_process_results
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+generation_kwargs:
+  max_new_tokens: 16
+model_specific_generation_kwargs:
+  llava:
+    image_aspect_ratio: original
+metric_list:
+  - metric: mmmu_acc
+    aggregation: !function utils.mmmu_aggregate_results
+    higher_is_better: true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xmmmu/mmmu_Portuguese_val.yaml b/lmms_eval/tasks/xmmmu/mmmu_Portuguese_val.yaml
new file mode 100644
index 000000000..c0f505eb9
--- /dev/null
+++ b/lmms_eval/tasks/xmmmu/mmmu_Portuguese_val.yaml
@@ -0,0 +1,21 @@
+dataset_path: neulab/PangeaBench-xmmmu
+task: "mmmu_Portuguese_val"
+test_split: pt
+output_type: generate_until
+doc_to_visual: !function utils.mmmu_doc_to_visual
+doc_to_text: !function utils.mmmu_doc_to_text
+doc_to_target: "answer"
+# The return value of process_results will be used by metrics
+process_results: !function utils.mmmu_process_results
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+generation_kwargs:
+  max_new_tokens: 16
+model_specific_generation_kwargs:
+  llava:
+    image_aspect_ratio: original
+metric_list:
+  - metric: mmmu_acc
+    aggregation: !function utils.mmmu_aggregate_results
+    higher_is_better: true
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/xmmmu/utils.py b/lmms_eval/tasks/xmmmu/utils.py
new file mode 100644
index 000000000..04c2edb7b
--- /dev/null
+++ b/lmms_eval/tasks/xmmmu/utils.py
@@ -0,0 +1,480 @@
+from collections import defaultdict
+import re
+import ast
+import random
+import numpy as np
+import os
+import json
+import logging
+
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+
+lmms_logger = logging.getLogger("lmms-eval")
+
+MULTI_CHOICE_PROMPT = "Answer with the option letter from the given choices directly."
+OPEN_ENDED_PROMPT = "Answer the question using a single word or phrase."
+
+
+def replace_images_tokens(input_string):
+    for i in range(1, 8):
+        question_text = f"<image {i}>"
+        query_text = "<image>"
+        if question_text in input_string:
+            input_string = input_string.replace(question_text, query_text)
+    return input_string
+
+
+def parse_options(options):
+    option_letters = [chr(ord("A") + i) for i in range(len(options))]
+    choices_str = "\n".join([f"{option_letter}. {option}" for option_letter, option in zip(option_letters, options)])
+    return choices_str
+
+
+def construct_prompt(doc):
+    question = doc["question"]
+    if doc["question_type"] == "multiple-choice":
+        # Weirdly, data["options"] is a string in MMMU Huggingface dataset
+        parsed_options = parse_options(ast.literal_eval(doc["options"].replace('\n', ' ')))  ######################################### there are some syntax errors
+        # parsed_options already prepends a newline so no need to add space here
+        question = f"{question}\n{parsed_options}\n{MULTI_CHOICE_PROMPT}"
+    else:
+        question = f"{question}\n{OPEN_ENDED_PROMPT}"
+    return question
+
+
+def mmmu_doc_to_text(doc):
+    question = construct_prompt(doc)
+    return replace_images_tokens(question)
+
+from PIL import Image
+import base64
+from io import BytesIO
+def base64_to_pil_image(base64_string):
+    img_bytes = base64.b64decode(base64_string)
+    
+    buffered = BytesIO(img_bytes)
+    
+    image = Image.open(buffered)
+    # image.save('temp.png')
+    return image
+
+def mmmu_doc_to_visual(doc):
+    prompt = construct_prompt(doc)
+    image_tokens = re.findall(r"<image \d+>", prompt)
+    # Remove <> and  swap space as _
+    image_tokens = [image_token.strip("<>").replace(" ", "_") for image_token in image_tokens]
+    visual = [doc[image_token].convert("RGB") for image_token in image_tokens if doc[image_token] is not None]   ######################################### load image from base64 encoding
+    return visual
+
+
+def mmmu_process_results(doc, results):
+    pred = results[0]
+    if doc["question_type"] == "multiple-choice":
+        index2ans, all_choices = get_multi_choice_info(ast.literal_eval(doc["options"].replace('\n', ' ')))  ######################################### there are some syntax errors
+        parsed_pred = parse_multi_choice_response(pred, all_choices, index2ans)
+    else:
+        parsed_pred = parse_open_response(pred)
+    id = doc["id"]
+    mmmu_acc = {"id": id, "subdomain": extract_subset_name(doc["id"]), "question_type": doc["question_type"], "answer": doc["answer"], "parsed_pred": parsed_pred}
+    return {
+        "mmmu_acc": mmmu_acc,
+        "submission": {
+            id: pred,
+        },
+    }
+
+
+def extract_subset_name(input_string):
+    # Define a regex pattern to match "validation_" at the beginning and "_<number>" at the end
+    split = input_string.split("_")[0]
+    pattern = re.compile(rf"^{split}_(.+?)_\d+$")
+    match = pattern.search(input_string)
+    if match:
+        return match.group(1)
+    else:
+        raise ValueError(f'No match found in "{input_string}"')
+
+
+def mmmu_test_aggregate_results_for_submission(results, args):
+    path = generate_submission_file("mmmu_test_for_submission.json", args)
+    with open(path, "w") as f:
+        json.dump(results, f)
+    lmms_logger.info(f"Results saved to {path}.")
+
+
+def mmmu_aggregate_results(results):
+    evaluation_result = {}
+    subset_to_eval_samples = defaultdict(list)
+    for result in results:
+        subset_to_eval_samples[result["subdomain"]].append(result)
+    for subset, sub_eval_samples in subset_to_eval_samples.items():
+        judge_dict, metric_dict = evaluate_mmmu(sub_eval_samples)
+        metric_dict.update({"num_example": len(sub_eval_samples)})
+        evaluation_result[subset] = metric_dict
+    printable_results = {}
+    for domain, in_domain_cats in DOMAIN_CAT2SUB_CAT.items():
+        in_domain_cat_results = {}
+        for cat_name in in_domain_cats:
+            if cat_name in evaluation_result.keys():
+                in_domain_cat_results[cat_name] = evaluation_result[cat_name]
+            else:
+                pass
+        in_domain_ins_acc = calculate_ins_level_acc(in_domain_cat_results)
+        in_domain_data_num = sum([cat_results["num_example"] for cat_results in in_domain_cat_results.values()])
+        printable_results["Overall-" + domain] = {
+            "num": int(in_domain_data_num),
+            "acc": round(in_domain_ins_acc, 3),
+        }
+        # add sub category
+        for cat_name, cat_results in in_domain_cat_results.items():
+            printable_results[cat_name] = {
+                "num": int(cat_results["num_example"]),
+                "acc": round(cat_results["acc"], 3),
+            }
+    all_ins_acc = calculate_ins_level_acc(evaluation_result)
+    printable_results["Overall"] = {
+        "num": sum([cat_results["num_example"] for cat_results in evaluation_result.values()]),
+        "acc": round(all_ins_acc, 3),
+    }
+    print(printable_results)
+    return printable_results["Overall"]["acc"]
+
+
+##################
+# Helper functions written by official MMMU repo.
+##################
+
+
+def calculate_ins_level_acc(results):
+    """Calculate the instruction level accuracy for given Subject results
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L246
+    """
+    acc = 0
+    ins_num = 0
+    for cat_results in results.values():
+        acc += cat_results["acc"] * cat_results["num_example"]
+        ins_num += cat_results["num_example"]
+    if ins_num == 0:
+        return 0
+    return acc / ins_num
+
+
+DOMAIN_CAT2SUB_CAT = {
+    "Art and Design": ["Art", "Art_Theory", "Design", "Music"],
+    "Business": ["Accounting", "Economics", "Finance", "Manage", "Marketing"],
+    "Science": [
+        "Biology",
+        "Chemistry",
+        "Geography",
+        "Math",
+        "Physics",
+    ],
+    "Health and Medicine": [
+        "Basic_Medical_Science",
+        "Clinical_Medicine",
+        "Diagnostics_and_Laboratory_Medicine",
+        "Pharmacy",
+        "Public_Health",
+    ],
+    "Humanities and Social Science": [
+        "History",
+        "Literature",
+        "Sociology",
+        "Psychology",
+    ],
+    "Tech and Engineering": [
+        "Agriculture",
+        "Architecture_and_Engineering",
+        "Computer_Science",
+        "Electronics",
+        "Energy_and_Power",
+        "Materials",
+        "Mechanical_Engineering",
+    ],
+}
+
+
+def eval_multi_choice(gold_i, pred_i):
+    """
+    Evaluate a multiple choice instance.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L175
+    """
+    correct = False
+    # only they are exactly the same, we consider it as correct
+    if isinstance(gold_i, list):
+        for answer in gold_i:
+            if answer == pred_i:
+                correct = True
+                break
+    else:  # gold_i is a string
+        if gold_i == pred_i:
+            correct = True
+    return correct
+
+
+def eval_open(gold_i, pred_i):
+    """
+    Evaluate an open question instance
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L191
+    """
+    correct = False
+    if isinstance(gold_i, list):
+        # use float to avoid trivial matches
+        norm_answers = []
+        for answer in gold_i:
+            norm_answers.extend(normalize_str(answer))
+    else:
+        norm_answers = normalize_str(gold_i)
+    for pred in pred_i:  # pred is already normalized in parse response phase
+        if isinstance(pred, str):  # if it's a string, then find if ans in the pred_i
+            for norm_ans in norm_answers:
+                # only see if the string answer in the string pred
+                if isinstance(norm_ans, str) and norm_ans in pred:
+                    if not correct:
+                        correct = True
+                    break
+        else:  # it's a float number
+            if pred in norm_answers:
+                if not correct:
+                    correct = True
+                break
+    return correct
+
+
+def evaluate_mmmu(samples):
+    """
+    Batch evaluation for multiple choice and open questions.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L219
+    """
+    pred_correct = 0
+    judge_dict = dict()
+    for sample in samples:
+        gold_i = sample["answer"]
+        pred_i = sample["parsed_pred"]
+        if sample["question_type"] == "multiple-choice":
+            correct = eval_multi_choice(gold_i, pred_i)
+        else:  # open question
+            correct = eval_open(gold_i, pred_i)
+
+        if correct:
+            judge_dict[sample["id"]] = "Correct"
+            pred_correct += 1
+        else:
+            judge_dict[sample["id"]] = "Wrong"
+
+    if len(samples) == 0:
+        return {"acc": 0}
+    return judge_dict, {"acc": pred_correct / len(samples)}
+
+
+def parse_multi_choice_response(response, all_choices, index2ans):
+    """
+    Parse the prediction from the generated response.
+    Return the predicted index e.g., A, B, C, D.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L10
+    """
+    for char in [",", ".", "!", "?", ";", ":", "'"]:
+        response = response.strip(char)
+    response = " " + response + " "  # add space to avoid partial match
+
+    index_ans = True
+    ans_with_brack = False
+    candidates = []
+    for choice in all_choices:  # e.g., (A) (B) (C) (D)
+        if f"({choice})" in response:
+            candidates.append(choice)
+            ans_with_brack = True
+
+    if len(candidates) == 0:
+        for choice in all_choices:  # e.g., A B C D
+            if f"{choice} " in response:
+                candidates.append(choice)
+
+    if len(candidates) == 0:
+        for choice in all_choices:  # e.g., A. B. C. D.
+            if f"{choice}." in response:
+                candidates.append(choice)
+
+    # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example
+    if len(candidates) == 0 and len(response.split()) > 5:
+        for index, ans in index2ans.items():
+            if ans.lower() in response.lower():
+                candidates.append(index)
+                index_ans = False  # it's content ans.
+
+    if len(candidates) == 0:  # still not get answer, randomly choose one.
+        pred_index = random.choice(all_choices)
+    elif len(candidates) > 1:
+        start_indexes = []
+        if index_ans:
+            if ans_with_brack:
+                for can in candidates:
+                    index = response.rfind(f"({can})")
+                    start_indexes.append(index)  # -1 will be ignored anyway
+                # start_indexes = [generated_response.index(f'({can})') for can in candidates]
+            else:
+                for can in candidates:
+                    index = response.rfind(f" {can} ")
+                    start_indexes.append(index)
+        else:
+            for can in candidates:
+                index = response.lower().rfind(index2ans[can].lower())
+                start_indexes.append(index)
+        # get the last one
+        pred_index = candidates[np.argmax(start_indexes)]
+    else:  # if only one candidate, use it.
+        pred_index = candidates[0]
+
+    return pred_index
+
+
+def extract_numbers(string):
+    """
+    Exact all forms of numbers from a string with regex.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L100
+    """
+    # Pattern for numbers with commas
+    pattern_commas = r"-?\b\d{1,3}(?:,\d{3})+\b"
+    # Pattern for scientific notation
+    pattern_scientific = r"-?\d+(?:\.\d+)?[eE][+-]?\d+"
+    # Pattern for simple numbers without commas
+    pattern_simple = r"-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])"
+
+    # Extract numbers with commas
+    numbers_with_commas = re.findall(pattern_commas, string)
+    # Extract numbers in scientific notation
+    numbers_scientific = re.findall(pattern_scientific, string)
+    # Extract simple numbers without commas
+    numbers_simple = re.findall(pattern_simple, string)
+
+    # Combine all extracted numbersz
+    all_numbers = numbers_with_commas + numbers_scientific + numbers_simple
+    return all_numbers
+
+
+def check_is_number(string):
+    """
+    Check if the given string a number.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L65
+    """
+    try:
+        float(string.replace(",", ""))
+        return True
+    except ValueError:
+        # check if there's comma inside
+        return False
+
+
+def normalize_str(string):
+    """
+    Normalize the str to lower case and make them float numbers if possible.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L76
+    """
+    # check if characters in the string
+
+    # if number, numerize it.
+    string = string.strip()
+
+    is_number = check_is_number(string)
+
+    if is_number:
+        string = string.replace(",", "")
+        string = float(string)
+        # leave 2 decimal
+        string = round(string, 2)
+        return [string]
+    else:  # it's likely to be a string
+        # lower it
+        string = string.lower()
+        if len(string) == 1:
+            return [" " + string, string + " "]  # avoid trivial matches
+        return [string]
+
+
+def parse_open_response(response):
+    """
+    Parse the prediction from the generated response.
+    Return a list of predicted strings or numbers.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L122
+    """
+
+    # content = content.strip("\n").strip(".").strip(" ")
+    def get_key_subresponses(response):
+        key_responses = []
+        response = response.strip().strip(".").lower()
+        sub_responses = re.split(r"\.\s(?=[A-Z])|\n", response)
+        indicators_of_keys = [
+            "could be ",
+            "so ",
+            "is ",
+            "thus ",
+            "therefore ",
+            "final ",
+            "answer ",
+            "result ",
+        ]
+        key_responses = []
+        for index, resp in enumerate(sub_responses):
+            # if last one, accept it's an equation (the entire response can be just one sentence with equation)
+            if index == len(sub_responses) - 1:
+                indicators_of_keys.extend(["="])
+            shortest_key_response = None  # the shortest response that may contain the answer (tail part of the response)
+            for indicator in indicators_of_keys:
+                if indicator in resp:
+                    if not shortest_key_response:
+                        shortest_key_response = resp.split(indicator)[-1].strip()
+                    else:
+                        if len(resp.split(indicator)[-1].strip()) < len(shortest_key_response):
+                            shortest_key_response = resp.split(indicator)[-1].strip()
+                    # key_responses.append(resp.split(indicator)[1].strip())
+
+            if shortest_key_response:
+                # and it's not trivial
+                if shortest_key_response.strip() not in [
+                    ":",
+                    ",",
+                    ".",
+                    "!",
+                    "?",
+                    ";",
+                    ":",
+                    "'",
+                ]:
+                    key_responses.append(shortest_key_response)
+        if len(key_responses) == 0:  # did not found any
+            return [response]
+        return key_responses
+
+    # pdb.set_trace()
+    key_responses = get_key_subresponses(response)
+
+    pred_list = key_responses.copy()  # keep the original string response
+    for resp in key_responses:
+        pred_list.extend(extract_numbers(resp))
+
+    tmp_pred_list = []
+    for i in range(len(pred_list)):
+        tmp_pred_list.extend(normalize_str(pred_list[i]))
+    pred_list = tmp_pred_list
+
+    # remove duplicates
+    pred_list = list(set(pred_list))
+
+    return pred_list
+
+
+def get_multi_choice_info(options):
+    """
+    Given the list of options for multiple choice question
+    Return the index2ans and all_choices
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/data_utils.py#L54
+    """
+
+    start_chr = "A"
+    all_choices = []
+    index2ans = {}
+    for i, option in enumerate(options):
+        index2ans[chr(ord(start_chr) + i)] = option
+        all_choices.append(chr(ord(start_chr) + i))
+
+    return index2ans, all_choices
\ No newline at end of file
diff --git a/lmms_eval/tasks/xmmmu/xmmmu.yaml b/lmms_eval/tasks/xmmmu/xmmmu.yaml
new file mode 100644
index 000000000..235e52d2b
--- /dev/null
+++ b/lmms_eval/tasks/xmmmu/xmmmu.yaml
@@ -0,0 +1,13 @@
+group: xmmmu
+task:
+- mmmu_Arabic_val
+- mmmu_French_val
+- mmmu_English_val
+- mmmu_Hindi_val
+- mmmu_Indonesian_val
+- mmmu_Japanese_val
+- mmmu_Portuguese_val
+metric_list:
+  - metric: mmmu_acc
+    aggregation: !function utils.mmmu_aggregate_results
+    higher_is_better: true
\ No newline at end of file
diff --git a/run_eval.sh b/run_eval.sh
index 8709e5c66..f3dea173e 100755
--- a/run_eval.sh
+++ b/run_eval.sh
@@ -1,17 +1,49 @@
+#!/bin/bash
+
+#SBATCH --job-name=pangea_eval   # Job name
+#SBATCH --time=48:00:00         # Run time (hh:mm:ss) 
+#SBATCH --gres=gpu:2           # Number of GPUs to be used
+#SBATCH --qos=gpu-medium         # QOS to be used
+#SBATCH --partition=a6000         # QOS to be used
+#SBATCH --mem=500GB         # mem to use
+#SBATCH --output=/mnt/data-poseidon/manos/tower-vision-eval/deepspin-lmms-eval/%x_%j.out  # Standard output
+#SBATCH --error=/mnt/data-poseidon/manos/tower-vision-eval/deepspin-lmms-eval/%x_%j.err   # Standard error
+
+
+source ~/.bashrc
+
 out_root=/mnt/scratch-artemis/manos/data/tower-vision-eval-outputs
+env_path=/mnt/data-poseidon/manos/tower-vision-eval/deepspin-lmms-eval/venv-lmms-eval
+
+
+module load openjdk
+# Activate the environment
+source $env_path/bin/activate
+
+mkdir -p $out_root/outs3/
 
-mkdir -p $out_root/outs1/
+# python -m accelerate.commands.launch \
+#     --num_processes=1 \
+#     -m lmms_eval \
+#     --model molmo_hf \
+#     --model_args pretrained='allenai/Molmo-7B-D-0924' \
+#     --tasks mmmu,mme,scienceqa \
+#     --batch_size 1 \
+#     --log_samples \
+#     --log_samples_suffix llava_next \
+#     --output_path $out_root/outs1/
 
-python3 -m accelerate.commands.launch \
-    --num_processes=1 \
+python -m accelerate.commands.launch \
+    --num_processes=2 \
     -m lmms_eval \
-    --model molmo_hf \
-    --model_args pretrained='allenai/Molmo-7B-D-0924' \
-    --tasks mmmu,mme,scienceqa \
+    --model pangea  \
+    --model_args pretrained="neulab/Pangea-7B",device_map=auto \
+    --tasks cvqa \
     --batch_size 1 \
-    --log_samples \
-    --log_samples_suffix llava_next \
-    --output_path $out_root/outs1/
+    --output_path $out_root/outs3/ \
+    --log_samples 
+    # --log_samples_suffix llava_next \
+# --tasks xmmmu,marvl,m3exam,maxm,xgqa \
 
     # --model_args pretrained="Unbabel/qwen2p5-7b-clip-hdr-sft-v3",add_system_prompt="Answer the questions." \
     # --model_args pretrained="Unbabel/qwen2p5-7b-clip-hdr-sft-v3",add_system_prompt="Answer the questions." \
diff --git a/run_eval_aggregate.py b/run_eval_aggregate.py
new file mode 100644
index 000000000..f2b20ae7e
--- /dev/null
+++ b/run_eval_aggregate.py
@@ -0,0 +1,100 @@
+import json
+import argparse
+import re 
+import os
+import numpy as np
+from typing import Dict
+
+
+ENGLISH_TASKS = {
+    "marvl":"nlvr2",
+    "m3exam":"m3exam_english",
+    "maxm":"maxm_en",
+    "xm100":"xm100_en",
+    "xmmmu":"mmmu_English_val",
+    "xgqa":"xgqa_en"
+}
+
+
+def parse_arguments():
+    """
+    Parse command-line arguments using argparse.
+    Returns:
+        Namespace: Parsed arguments containing input_file and output_file paths.
+    """
+    parser = argparse.ArgumentParser(description="Calculate the average metrics from a JSON file.")
+    parser.add_argument("--inputfile", type=str, help="Path to the input JSON file.")
+    parser.add_argument("--outputdir", type=str, help="Path to the output dir.")
+    return parser.parse_args()
+
+def aggregate_results(data):
+    
+    subtasks_dict = data["group_subtasks"]
+    pattern = r"^[a-z0-9_]+(?<!_stderr)(?=,none$)"
+    pattern = r"^(?!.*_stderr)[^,]+(?=,none$)"
+
+    subtask_collect_multi_results={maintask_name:{} for maintask_name in subtasks_dict }
+    subtask_collect_eng_results={maintask_name:{} for maintask_name in subtasks_dict }
+    for maintask_name in subtasks_dict:
+        main_task_dict_multi = {}
+        main_task_dict_eng = {}
+        for subtask in subtasks_dict[maintask_name]:
+            draft_dict = data["results"][subtask]
+            metric_names = [re.match(pattern, field).group() for field in draft_dict if re.match(pattern, field)]
+            if subtask==ENGLISH_TASKS[maintask_name]:
+                for metric in metric_names:
+                    value = draft_dict[f"{metric},none"]
+                    if metric not in main_task_dict_eng:
+                        main_task_dict_eng[metric] = [value]
+                    else:
+                        main_task_dict_eng[metric].append(value)
+            else:
+                for metric in metric_names:
+                    value = draft_dict[f"{metric},none"]
+                    if metric not in main_task_dict_multi:
+                        main_task_dict_multi[metric] = [value]
+                    else:
+                        main_task_dict_multi[metric].append(value)
+        
+        subtask_collect_multi_results[maintask_name] = main_task_dict_multi
+        subtask_collect_eng_results[maintask_name] = main_task_dict_eng
+    
+    aggregated_results_multi = {}
+    aggregated_results_eng = {}
+    for task_name in subtask_collect_multi_results:
+        aggregated_results_multi[task_name] = {metric:np.mean(subtask_collect_multi_results[task_name][metric]) for metric in subtask_collect_multi_results[task_name] if metric!="submission"}
+        aggregated_results_eng[task_name] = {metric:np.mean(subtask_collect_eng_results[task_name][metric]) for metric in subtask_collect_eng_results[task_name] if metric!="submission"}
+        
+    return aggregated_results_eng, aggregated_results_multi
+
+
+def main():
+    # Parse arguments
+    args = parse_arguments()
+
+    # Load input JSON
+    with open(args.inputfile, "r") as infile:
+        data = json.load(infile)
+
+
+    results_eng, results_multi = aggregate_results(data)
+
+    # Prepare output directory
+    os.makedirs(args.outputdir, exist_ok=True)
+
+    # Write English task results to JSON file
+    eng_output_path = os.path.join(args.outputdir, "english_tasks_results.json")
+    with open(eng_output_path, "w") as eng_outfile:
+        json.dump(results_eng, eng_outfile, indent=4)
+
+    # Write multi-task results to JSON file
+    multi_output_path = os.path.join(args.outputdir, "multi_tasks_results.json")
+    with open(multi_output_path, "w") as multi_outfile:
+        json.dump(results_multi, multi_outfile, indent=4)
+
+    print(f"English task results saved to {eng_output_path}.")
+    print(f"Multi-task results saved to {multi_output_path}.")
+
+
+if __name__ == "__main__":
+    main()

From d944f971b13e04a96b68dad716954e101a410e96 Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel-8a100.datacrunch.io>
Date: Tue, 7 Jan 2025 09:21:53 +0000
Subject: [PATCH 006/112] small fix for nvlm_d class and config files of maxm
 task

---
 lmms_eval/models/nvlm_d.py        | 9 ++++++---
 lmms_eval/tasks/maxm/maxm_en.yaml | 2 +-
 lmms_eval/tasks/maxm/maxm_fr.yaml | 2 +-
 lmms_eval/tasks/maxm/maxm_hi.yaml | 2 +-
 lmms_eval/tasks/maxm/maxm_iw.yaml | 2 +-
 lmms_eval/tasks/maxm/maxm_ro.yaml | 2 +-
 lmms_eval/tasks/maxm/maxm_th.yaml | 2 +-
 lmms_eval/tasks/maxm/maxm_zh.yaml | 2 +-
 8 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/lmms_eval/models/nvlm_d.py b/lmms_eval/models/nvlm_d.py
index c41f48577..9c607e9cd 100644
--- a/lmms_eval/models/nvlm_d.py
+++ b/lmms_eval/models/nvlm_d.py
@@ -50,12 +50,15 @@ def __init__(
         assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
 
         accelerator = Accelerator()
-        if accelerator.num_processes > 1 and device_map == "":
+        if accelerator.num_processes > 1:
             self._device = torch.device(f"cuda:{accelerator.local_process_index}")
             self.device_map = f"cuda:{accelerator.local_process_index}"
-        else:
+        elif accelerator.num_processes == 1 and device_map == "auto":
             self._device = torch.device(device)
             self.device_map = device_map
+        else:
+            self._device = torch.device(f"cuda:{accelerator.local_process_index}")
+            self.device_map = f"cuda:{accelerator.local_process_index}"
 
         if isinstance(dtype, str) and dtype != "auto":
             dtype = getattr(torch, dtype)
@@ -80,7 +83,7 @@ def __init__(
         self.use_cache = use_cache
         self.add_system_prompt = add_system_prompt
         # Handle distributed setup
-        if accelerator.num_processes > 1 and device_map == "":
+        if accelerator.num_processes > 1:
             assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED]
             
             if accelerator.distributed_type == DistributedType.DEEPSPEED:
diff --git a/lmms_eval/tasks/maxm/maxm_en.yaml b/lmms_eval/tasks/maxm/maxm_en.yaml
index 0e159b949..74a62821b 100755
--- a/lmms_eval/tasks/maxm/maxm_en.yaml
+++ b/lmms_eval/tasks/maxm/maxm_en.yaml
@@ -28,7 +28,7 @@ metric_list:
   - metric: relaxed_accuracy
     aggregation: !function utils.maxm_relaxed_ema
     higher_is_better: true
-model_specific_prompt_kwargs:
+lmms_eval_specific_kwargs:
   default:
     pre_prompt: "Given the image and the question, provide a concise answer. "
     post_prompt: "\nAnswer the question using a single word or phrase in English."
diff --git a/lmms_eval/tasks/maxm/maxm_fr.yaml b/lmms_eval/tasks/maxm/maxm_fr.yaml
index 26592f56d..1b396ce34 100755
--- a/lmms_eval/tasks/maxm/maxm_fr.yaml
+++ b/lmms_eval/tasks/maxm/maxm_fr.yaml
@@ -28,7 +28,7 @@ metric_list:
   - metric: relaxed_accuracy
     aggregation: !function utils.maxm_relaxed_ema
     higher_is_better: true
-model_specific_prompt_kwargs:
+lmms_eval_specific_kwargs:
   default:
     pre_prompt: "Given the image and the question, provide a concise answer. "
     post_prompt: "\nAnswer the question using a single word or phrase in French."
diff --git a/lmms_eval/tasks/maxm/maxm_hi.yaml b/lmms_eval/tasks/maxm/maxm_hi.yaml
index 7d4fde6e9..81e2398de 100755
--- a/lmms_eval/tasks/maxm/maxm_hi.yaml
+++ b/lmms_eval/tasks/maxm/maxm_hi.yaml
@@ -28,7 +28,7 @@ metric_list:
   - metric: relaxed_accuracy
     aggregation: !function utils.maxm_relaxed_ema
     higher_is_better: true
-model_specific_prompt_kwargs:
+lmms_eval_specific_kwargs:
   default:
     pre_prompt: "Given the image and the question, provide a concise answer. "
     post_prompt: "\nAnswer the question using a single word or phrase in Hindi."
diff --git a/lmms_eval/tasks/maxm/maxm_iw.yaml b/lmms_eval/tasks/maxm/maxm_iw.yaml
index be25cb772..9d963d33c 100755
--- a/lmms_eval/tasks/maxm/maxm_iw.yaml
+++ b/lmms_eval/tasks/maxm/maxm_iw.yaml
@@ -28,7 +28,7 @@ metric_list:
   - metric: relaxed_accuracy
     aggregation: !function utils.maxm_relaxed_ema
     higher_is_better: true
-model_specific_prompt_kwargs:
+lmms_eval_specific_kwargs:
   default:
     pre_prompt: "Given the image and the question, provide a concise answer. "
     post_prompt: "\nAnswer the question using a single word or phrase in Hebrew."
diff --git a/lmms_eval/tasks/maxm/maxm_ro.yaml b/lmms_eval/tasks/maxm/maxm_ro.yaml
index 96ed424f8..79f8b81e0 100755
--- a/lmms_eval/tasks/maxm/maxm_ro.yaml
+++ b/lmms_eval/tasks/maxm/maxm_ro.yaml
@@ -28,7 +28,7 @@ metric_list:
   - metric: relaxed_accuracy
     aggregation: !function utils.maxm_relaxed_ema
     higher_is_better: true
-model_specific_prompt_kwargs:
+lmms_eval_specific_kwargs:
   default:
     pre_prompt: "Given the image and the question, provide a concise answer. "
     post_prompt: "\nAnswer the question using a single word or phrase in Romanian."
diff --git a/lmms_eval/tasks/maxm/maxm_th.yaml b/lmms_eval/tasks/maxm/maxm_th.yaml
index d0935c6b2..df9f19508 100755
--- a/lmms_eval/tasks/maxm/maxm_th.yaml
+++ b/lmms_eval/tasks/maxm/maxm_th.yaml
@@ -28,7 +28,7 @@ metric_list:
   - metric: relaxed_accuracy
     aggregation: !function utils.maxm_relaxed_ema
     higher_is_better: true
-model_specific_prompt_kwargs:
+lmms_eval_specific_kwargs:
   default:
     pre_prompt: "Given the image and the question, provide a concise answer. "
     post_prompt: "\nAnswer the question using a single word or phrase in Thai."
diff --git a/lmms_eval/tasks/maxm/maxm_zh.yaml b/lmms_eval/tasks/maxm/maxm_zh.yaml
index 599a247a9..ab4397883 100755
--- a/lmms_eval/tasks/maxm/maxm_zh.yaml
+++ b/lmms_eval/tasks/maxm/maxm_zh.yaml
@@ -28,7 +28,7 @@ metric_list:
   - metric: relaxed_accuracy
     aggregation: !function utils.maxm_relaxed_ema
     higher_is_better: true
-model_specific_prompt_kwargs:
+lmms_eval_specific_kwargs:
   default:
     pre_prompt: "Given the image and the question, provide a concise answer. "
     post_prompt: "\nAnswer the question using a single word or phrase in Chinese."

From 0a4246c1afe26cea1bfed3890646f0e1c01c8bba Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Tue, 7 Jan 2025 10:56:05 +0000
Subject: [PATCH 007/112] pangea update+ updates on maxm task

---
 lmms_eval/models/pangea.py        | 21 +++++++++++----------
 lmms_eval/tasks/maxm/maxm_en.yaml |  2 +-
 lmms_eval/tasks/maxm/maxm_fr.yaml |  2 +-
 lmms_eval/tasks/maxm/maxm_hi.yaml |  2 +-
 lmms_eval/tasks/maxm/maxm_iw.yaml |  2 +-
 lmms_eval/tasks/maxm/maxm_ro.yaml |  2 +-
 lmms_eval/tasks/maxm/maxm_th.yaml |  2 +-
 lmms_eval/tasks/maxm/maxm_zh.yaml |  2 +-
 run_eval.sh                       |  8 ++++----
 9 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/lmms_eval/models/pangea.py b/lmms_eval/models/pangea.py
index 768fc87eb..88a1e1eb8 100644
--- a/lmms_eval/models/pangea.py
+++ b/lmms_eval/models/pangea.py
@@ -53,7 +53,7 @@ def __init__(
         truncation: Optional[bool] = True,
         device: Optional[str] = "cuda:0",
         batch_size: Optional[Union[int, str]] = 1,
-        model_name="pangea-7b-llava-clip-qwen2",
+        model_name='Pangea-7B-qwen', #'Pangea-7B-qwen' recommened in https://huggingface.co/neulab/Pangea-7B , # pangea repo sets this to None, I was using "pangea-7b-llava-clip-qwen2"
         attn_implementation=best_fit_attn_implementation,
         device_map="cuda:0",
         conv_template="qwen_1_5",
@@ -80,25 +80,26 @@ def __init__(
             self._device = torch.device(f"cuda:{accelerator.local_process_index}")
             self.device_map = f"cuda:{accelerator.local_process_index}"
 
-        llava_model_args = {
+        model_args = {
             "multimodal": True,
         }
         if customized_config is not None:
-            llava_model_args["customized_config"] = customized_config
+            model_args["customized_config"] = customized_config
         if attn_implementation is not None:
-            llava_model_args["attn_implementation"] = attn_implementation
+            model_args["attn_implementation"] = attn_implementation
         if "use_flash_attention_2" in kwargs:
-            llava_model_args["use_flash_attention_2"] = kwargs["use_flash_attention_2"]
+            model_args["use_flash_attention_2"] = kwargs["use_flash_attention_2"]
         self.model_name = model_name
-        # model_name = model_name if model_name is not None else get_model_name_from_path(pretrained)
-        # model_name = "llava-v1.5-7b" if pretrained == "MBZUAI/PALO-7B" else get_model_name_from_path(pretrained)
+
         try:
             # Try to load the model with the multimodal argument
-            self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
+            self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, **model_args)
+
         except TypeError:
             # for older versions of LLaVA that don't have multimodal argument
-            llava_model_args.pop("multimodal", None)
-            self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
+            model_args.pop("multimodal", None)
+            self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **model_args)
+        
         self._config = self._model.config
         self.model.eval()
         if tie_weights:
diff --git a/lmms_eval/tasks/maxm/maxm_en.yaml b/lmms_eval/tasks/maxm/maxm_en.yaml
index 0e159b949..74a62821b 100755
--- a/lmms_eval/tasks/maxm/maxm_en.yaml
+++ b/lmms_eval/tasks/maxm/maxm_en.yaml
@@ -28,7 +28,7 @@ metric_list:
   - metric: relaxed_accuracy
     aggregation: !function utils.maxm_relaxed_ema
     higher_is_better: true
-model_specific_prompt_kwargs:
+lmms_eval_specific_kwargs:
   default:
     pre_prompt: "Given the image and the question, provide a concise answer. "
     post_prompt: "\nAnswer the question using a single word or phrase in English."
diff --git a/lmms_eval/tasks/maxm/maxm_fr.yaml b/lmms_eval/tasks/maxm/maxm_fr.yaml
index 26592f56d..1b396ce34 100755
--- a/lmms_eval/tasks/maxm/maxm_fr.yaml
+++ b/lmms_eval/tasks/maxm/maxm_fr.yaml
@@ -28,7 +28,7 @@ metric_list:
   - metric: relaxed_accuracy
     aggregation: !function utils.maxm_relaxed_ema
     higher_is_better: true
-model_specific_prompt_kwargs:
+lmms_eval_specific_kwargs:
   default:
     pre_prompt: "Given the image and the question, provide a concise answer. "
     post_prompt: "\nAnswer the question using a single word or phrase in French."
diff --git a/lmms_eval/tasks/maxm/maxm_hi.yaml b/lmms_eval/tasks/maxm/maxm_hi.yaml
index 7d4fde6e9..81e2398de 100755
--- a/lmms_eval/tasks/maxm/maxm_hi.yaml
+++ b/lmms_eval/tasks/maxm/maxm_hi.yaml
@@ -28,7 +28,7 @@ metric_list:
   - metric: relaxed_accuracy
     aggregation: !function utils.maxm_relaxed_ema
     higher_is_better: true
-model_specific_prompt_kwargs:
+lmms_eval_specific_kwargs:
   default:
     pre_prompt: "Given the image and the question, provide a concise answer. "
     post_prompt: "\nAnswer the question using a single word or phrase in Hindi."
diff --git a/lmms_eval/tasks/maxm/maxm_iw.yaml b/lmms_eval/tasks/maxm/maxm_iw.yaml
index be25cb772..9d963d33c 100755
--- a/lmms_eval/tasks/maxm/maxm_iw.yaml
+++ b/lmms_eval/tasks/maxm/maxm_iw.yaml
@@ -28,7 +28,7 @@ metric_list:
   - metric: relaxed_accuracy
     aggregation: !function utils.maxm_relaxed_ema
     higher_is_better: true
-model_specific_prompt_kwargs:
+lmms_eval_specific_kwargs:
   default:
     pre_prompt: "Given the image and the question, provide a concise answer. "
     post_prompt: "\nAnswer the question using a single word or phrase in Hebrew."
diff --git a/lmms_eval/tasks/maxm/maxm_ro.yaml b/lmms_eval/tasks/maxm/maxm_ro.yaml
index 96ed424f8..79f8b81e0 100755
--- a/lmms_eval/tasks/maxm/maxm_ro.yaml
+++ b/lmms_eval/tasks/maxm/maxm_ro.yaml
@@ -28,7 +28,7 @@ metric_list:
   - metric: relaxed_accuracy
     aggregation: !function utils.maxm_relaxed_ema
     higher_is_better: true
-model_specific_prompt_kwargs:
+lmms_eval_specific_kwargs:
   default:
     pre_prompt: "Given the image and the question, provide a concise answer. "
     post_prompt: "\nAnswer the question using a single word or phrase in Romanian."
diff --git a/lmms_eval/tasks/maxm/maxm_th.yaml b/lmms_eval/tasks/maxm/maxm_th.yaml
index d0935c6b2..df9f19508 100755
--- a/lmms_eval/tasks/maxm/maxm_th.yaml
+++ b/lmms_eval/tasks/maxm/maxm_th.yaml
@@ -28,7 +28,7 @@ metric_list:
   - metric: relaxed_accuracy
     aggregation: !function utils.maxm_relaxed_ema
     higher_is_better: true
-model_specific_prompt_kwargs:
+lmms_eval_specific_kwargs:
   default:
     pre_prompt: "Given the image and the question, provide a concise answer. "
     post_prompt: "\nAnswer the question using a single word or phrase in Thai."
diff --git a/lmms_eval/tasks/maxm/maxm_zh.yaml b/lmms_eval/tasks/maxm/maxm_zh.yaml
index 599a247a9..ab4397883 100755
--- a/lmms_eval/tasks/maxm/maxm_zh.yaml
+++ b/lmms_eval/tasks/maxm/maxm_zh.yaml
@@ -28,7 +28,7 @@ metric_list:
   - metric: relaxed_accuracy
     aggregation: !function utils.maxm_relaxed_ema
     higher_is_better: true
-model_specific_prompt_kwargs:
+lmms_eval_specific_kwargs:
   default:
     pre_prompt: "Given the image and the question, provide a concise answer. "
     post_prompt: "\nAnswer the question using a single word or phrase in Chinese."
diff --git a/run_eval.sh b/run_eval.sh
index f3dea173e..f0fbeb07a 100755
--- a/run_eval.sh
+++ b/run_eval.sh
@@ -20,7 +20,7 @@ module load openjdk
 # Activate the environment
 source $env_path/bin/activate
 
-mkdir -p $out_root/outs3/
+mkdir -p $out_root/outs_official/
 
 # python -m accelerate.commands.launch \
 #     --num_processes=1 \
@@ -37,10 +37,10 @@ python -m accelerate.commands.launch \
     --num_processes=2 \
     -m lmms_eval \
     --model pangea  \
-    --model_args pretrained="neulab/Pangea-7B",device_map=auto \
-    --tasks cvqa \
+    --model_args pretrained="neulab/Pangea-7B",model_name='Pangea-7B-qwen',device_map=auto \
+    --tasks xmmmu,marvl,m3exam,maxm,xgqa,xm100 \
     --batch_size 1 \
-    --output_path $out_root/outs3/ \
+    --output_path $out_root/outs_official/ \
     --log_samples 
     # --log_samples_suffix llava_next \
 # --tasks xmmmu,marvl,m3exam,maxm,xgqa \

From cd6686b6bd18163e4050202857f16890c65ad391 Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Thu, 9 Jan 2025 15:53:02 +0000
Subject: [PATCH 008/112] modified output parsing for marvl

---
 lmms_eval/tasks/marvl/utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lmms_eval/tasks/marvl/utils.py b/lmms_eval/tasks/marvl/utils.py
index 5d1e06c19..777d4bd7c 100755
--- a/lmms_eval/tasks/marvl/utils.py
+++ b/lmms_eval/tasks/marvl/utils.py
@@ -20,6 +20,8 @@ def marvl_process_result(doc, results):
     pred = pred.strip().lower()
     if 'yes' in pred: pred = 'true'
     elif 'no' in pred: pred = 'false'
+    elif "(a)" in pred: pred = 'false'
+    elif "(b)" in pred: pred = 'true'
     if target.strip().lower() in pred:
         return {"exact_match": 1.0}
     return {"exact_match": 0.0}

From 9b1cd7a38215bc8b1a5fee53dd0ce97ea6dcb893 Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Fri, 10 Jan 2025 00:18:46 +0000
Subject: [PATCH 009/112] updated marvl task output parsing

---
 lmms_eval/tasks/marvl/utils.py | 52 +++++++++++++++++++++++++++++-----
 1 file changed, 45 insertions(+), 7 deletions(-)

diff --git a/lmms_eval/tasks/marvl/utils.py b/lmms_eval/tasks/marvl/utils.py
index 777d4bd7c..508d009c8 100755
--- a/lmms_eval/tasks/marvl/utils.py
+++ b/lmms_eval/tasks/marvl/utils.py
@@ -1,3 +1,4 @@
+from lmms_eval.utils import eval_logger
 
 def marvl_doc_to_visual(doc):
     return [doc["image"].convert("RGB")]
@@ -14,14 +15,51 @@ def marvl_doc_to_target(doc):
     elif 'false' in answer: return 'false'
     else: raise Exception(f"get target failed for id {doc['id']} - conversations: {conversations}")
 
+
+
+def check_output(doc,output):
+    accepted_answers = {
+    "(b)": "true",
+    "(a)": "false",
+    "false": "false",
+    "true": "true",
+    "false.": "false",
+    "true.": "true",
+    "b (b)": "true",
+    "a (a)": "false",
+    "b": "true",
+    "a": "false",
+    "b (true)": "true",
+    "a (false)": "false",
+    "b.": "true",
+    "a.": "false",
+    "a false": "false",
+    "b true": "true",
+    "a:": "false",
+    "b:": "true",
+    "(b) true": "true",
+    "(a) false": "false",
+    "yes":"true",
+    "no":"false",
+    "yes.":"true",
+    "no.":"false",
+    }
+    out = output.strip().lower()
+    if out in accepted_answers.keys():
+        return accepted_answers[out]
+    else:
+        eval_logger.warning("Model's output not in accepted answers. Doc: {doc}. Falling back to error...")
+        import ipdb;ipdb.set_trace()
+        return None
+
+
 def marvl_process_result(doc, results):
     target = marvl_doc_to_target(doc)
     pred = results[0]
-    pred = pred.strip().lower()
-    if 'yes' in pred: pred = 'true'
-    elif 'no' in pred: pred = 'false'
-    elif "(a)" in pred: pred = 'false'
-    elif "(b)" in pred: pred = 'true'
-    if target.strip().lower() in pred:
+    if target.strip().lower() not in ["true" , "false"]:
+        eval_logger.warning("Target '{target}' needs post-processing. Task: Marvl")
+    out = check_output(doc,pred)
+    if target.strip().lower()==out:
         return {"exact_match": 1.0}
-    return {"exact_match": 0.0}
+    else:
+        return {"exact_match": 0.0}

From 85f2c339b15cab04bb6e73b529eccaf11febeeed Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel.com>
Date: Mon, 10 Feb 2025 10:29:06 +0000
Subject: [PATCH 010/112] added the possibility of using tags during the
 inference

---
 lmms_eval/models/nvlm_d.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lmms_eval/models/nvlm_d.py b/lmms_eval/models/nvlm_d.py
index 9c607e9cd..72ebae9e1 100644
--- a/lmms_eval/models/nvlm_d.py
+++ b/lmms_eval/models/nvlm_d.py
@@ -42,6 +42,7 @@ def __init__(
         trust_remote_code: Optional[bool] = True,
         #attn_implementation: Optional[str] = best_fit_attn_implementation,
         add_system_prompt: Optional[str] = None,
+        tag: Optional[str] = None,
         device_map: str = "",
         use_cache: bool = True,
         **kwargs,
@@ -76,12 +77,12 @@ def __init__(
             revision=revision,
             trust_remote_code=trust_remote_code
         )
-
         self._tokenizer = self._processor.tokenizer
         self._config = self._model.config
         self.batch_size_per_gpu = int(batch_size)
         self.use_cache = use_cache
         self.add_system_prompt = add_system_prompt
+        self.tag = tag
         # Handle distributed setup
         if accelerator.num_processes > 1:
             assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED]
@@ -239,13 +240,14 @@ def _collate(x):
             
             if DEFAULT_IMAGE_TOKEN not in context:
                 context = f"{DEFAULT_IMAGE_TOKEN}\n{context}"
-                
+            
+            if self.tag:
+                context = f"{self.tag} " + context
             # create chat object and apply template
             chat = [{"role": "user", "content": context}]
             if self.add_system_prompt is not None:
                 chat.insert(0, {"role": "system", "content": self.add_system_prompt})
             prompt = self._tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)
-
             # Process inputs through the processor
             inputs = self._processor(images=[visual], text=[prompt], return_tensors="pt")
             inputs = {k: v.to(self.device) for k, v in inputs.items()}

From deaac0535382e0699f5befe10d5e4b529cfd5b19 Mon Sep 17 00:00:00 2001
From: aminfarajian <143111778+aminfarajian@users.noreply.github.com>
Date: Mon, 10 Feb 2025 10:46:20 +0000
Subject: [PATCH 011/112] Update .gitignore

---
 .gitignore | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index af4c8ea55..9dd92753e 100755
--- a/.gitignore
+++ b/.gitignore
@@ -19,7 +19,6 @@ temp
 profile_default/
 ipython_config.py
 logs/
-scripts/
 wandb/
 SimSun.ttf
 submissions/
@@ -43,4 +42,4 @@ VATEX/
 lmms_eval/tasks/vatex/__pycache__/utils.cpython-310.pyc
 lmms_eval/tasks/mlvu/__pycache__/utils.cpython-310.pyc
 
-scripts/
\ No newline at end of file
+scripts/

From 3ceee86a5e0d0e3b034d583c65b8093b7787e04d Mon Sep 17 00:00:00 2001
From: aminfarajian <143111778+aminfarajian@users.noreply.github.com>
Date: Mon, 10 Feb 2025 10:47:18 +0000
Subject: [PATCH 012/112] Update .gitignore

---
 .gitignore | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index 9dd92753e..a8d9cff41 100755
--- a/.gitignore
+++ b/.gitignore
@@ -41,5 +41,3 @@ Video-MME/
 VATEX/
 lmms_eval/tasks/vatex/__pycache__/utils.cpython-310.pyc
 lmms_eval/tasks/mlvu/__pycache__/utils.cpython-310.pyc
-
-scripts/

From 5c95b8f6600f2f98d06d1c398903eb79e0de426b Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel.com>
Date: Mon, 10 Feb 2025 10:52:32 +0000
Subject: [PATCH 013/112] added the scripts folder

---
 scripts/eval_runner_nvlm_d.sh | 48 +++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 scripts/eval_runner_nvlm_d.sh

diff --git a/scripts/eval_runner_nvlm_d.sh b/scripts/eval_runner_nvlm_d.sh
new file mode 100644
index 000000000..e8b2aafc5
--- /dev/null
+++ b/scripts/eval_runner_nvlm_d.sh
@@ -0,0 +1,48 @@
+rdir="/mnt/data/amin/NLP/widn/projects/tower-vision/results"
+
+tagged=false
+if [ "$tagged" = true ]; then
+    rdir="${rdir}.tagged"
+fi
+
+tasks=('ai2d' 'textvqa' 'mmmu')
+
+declare -A tags
+tags["ai2d"]="ai2d:"
+tags["textvqa"]="text_vqa:"
+tags["mmmu"]="figure_qa:"
+
+models=('Unbabel/qwen2p5-7b-hdr-sft-visionblocks-v0.4-tagged')
+for model in "${models[@]}"; do
+    echo "======================================================================================"
+    echo "model: $model"
+    for task in "${tasks[@]}"; do
+        echo "--------------------------------------------------------------------------------------"
+        echo "task: $task"
+        model_name=$(echo $model | sed 's/\//__/g')
+        results_folder=${rdir}/${task}/${model_name}
+
+        if ls ${results_folder}/*results.json 1> /dev/null 2>&1; then
+            echo "Results file already exists for $task"
+            continue
+        fi
+
+        model_args="${model}",add_system_prompt="\"Answer the questions.\"",device_map=auto
+
+        if [ "$tagged" = true ]; then
+            tag="${tags[$task]}"
+            model_args=${model_args},tag="${tag}"
+        fi
+        model_args="$model_args"
+        CUDA_VISIBLE_DEVICES=6,7 python -m accelerate.commands.launch \
+        --num_processes=2 \
+        -m lmms_eval \
+        --model nvlm_d \
+        --model_args pretrained="$model_args" \
+        --tasks $task \
+        --batch_size 1 \
+        --log_samples \
+        --log_samples_suffix $task \
+        --output_path $rdir/$task
+    done
+done
\ No newline at end of file

From 61349bbb7d31b0c874cfaccf2797f2e1a7217240 Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel.com>
Date: Mon, 10 Feb 2025 14:59:24 +0000
Subject: [PATCH 014/112] added the script to parse the results

---
 scripts/parse_results.py | 148 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 148 insertions(+)
 create mode 100644 scripts/parse_results.py

diff --git a/scripts/parse_results.py b/scripts/parse_results.py
new file mode 100644
index 000000000..2429a248c
--- /dev/null
+++ b/scripts/parse_results.py
@@ -0,0 +1,148 @@
+import argparse
+import json
+
+
+def get_results_m3exam(results):
+    score_en = results["m3exam_english"]["m3exam,none"]
+    results_multi = 0
+    results_multi = [
+        result["m3exam,none"]
+        for task, result in results.items()
+        if (task != "m3exam_english" and task != "m3exam")
+    ]
+    score_multi = sum(results_multi) / len(results_multi)
+    return score_en * 100, score_multi * 100
+
+
+def get_results_marvl(results):
+    score_en = results["nlvr2"]["exact_match,none"]
+    results_multi = 0
+    results_multi = [
+        result["exact_match,none"]
+        for task, result in results.items()
+        if (task != "marvl" and task != "nlvr2")
+    ]
+    score_multi = sum(results_multi) / len(results_multi)
+    return score_en * 100, score_multi * 100
+
+
+def get_results_maxm(results):
+    score_en = results["maxm_en"]["relaxed_accuracy,none"]
+    results_multi = 0
+    results_multi = [
+        result["relaxed_accuracy,none"]
+        for task, result in results.items()
+        if (task != "maxm" and task != "maxm_en")
+    ]
+    score_multi = sum(results_multi) / len(results_multi)
+    return score_en, score_multi
+
+
+def get_results_xgqa(results):
+    score_en = results["xgqa_en"]["exact_match,none"]
+    results_multi = 0
+    results_multi = [
+        result["exact_match,none"]
+        for task, result in results.items()
+        if (task != "xgqa" and task != "xgqa_en")
+    ]
+    score_multi = sum(results_multi) / len(results_multi)
+    return score_en * 100, score_multi * 100
+
+
+def get_results_xm100(results):
+    score_en = results["xm100_en"]["xm100_ROUGE_L,none"]
+    results_multi = 0
+    results_multi = [
+        result["xm100_ROUGE_L,none"]
+        for task, result in results.items()
+        if (task != "xm100" and task != "xm100_en")
+    ]
+    score_multi = sum(results_multi) / len(results_multi)
+    return score_en * 100, score_multi * 100
+
+
+def get_results_xmmmu(results):
+    score_en = results["mmmu_English_val"]["mmmu_acc,none"]
+    results_multi = 0
+    results_multi = [
+        result["mmmu_acc,none"]
+        for task, result in results.items()
+        if (task != "xmmmu" and task != "mmmu_English_val")
+    ]
+    score_multi = sum(results_multi) / len(results_multi)
+    return score_en * 100, score_multi * 100
+
+
+def get_results_multilingual_llava_bench(results):
+    score_en = results["multilingual_llava_bench_english"]["gpt_eval_llava_all,none"]
+    results_multi = 0
+    results_multi = [
+        result["gpt_eval_llava_all,none"]
+        for task, result in results.items()
+        if (
+            task != "multilingual_llava_bench"
+            and task != "multilingual_llava_bench_english"
+        )
+    ]
+    score_multi = sum(results_multi) / len(results_multi)
+    return score_en, score_multi
+
+
+def get_results_ai2d(results):
+    score_en = results["ai2d"]["exact_match,flexible-extract"]
+    return score_en * 100, None
+
+
+def get_results_mmmu(results):
+    score_en = results["mmmu_val_group_img"]["mmmu_acc,none"]
+    return score_en * 100, None
+
+
+def get_results_textvqa(results):
+    score_en = results["textvqa_val"]["exact_match,none"]
+    return score_en * 100, None
+
+def get_results_scienceqa(results):
+    score_en = results["scienceqa"]["exact_match,none"]
+    return score_en * 100, None
+
+def process_results(data, task):
+    results = data["results"]
+    score_en, score_multi = result_functions[task](results)
+    print(f"============== Results for {task} ==============")
+    if score_multi:
+        print(f"{score_en:.2f}\t{score_multi:.2f}")
+    else:
+        print(f"{score_en:.2f}")
+
+
+result_functions = {
+    "ai2d": get_results_ai2d,
+    "mmmu": get_results_mmmu,
+    "textvqa": get_results_textvqa,
+    "m3exam": get_results_m3exam,
+    "marvl": get_results_marvl,
+    "maxm": get_results_maxm,
+    "xgqa": get_results_xgqa,
+    "xm100": get_results_xm100,
+    "xmmmu": get_results_xmmmu,
+    "mllava_bench": get_results_multilingual_llava_bench,
+    "scienceqa": get_results_scienceqa,
+}
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Process a JSON file.")
+    parser.add_argument("--input_file", type=str, help="Path to the input JSON file")
+    parser.add_argument("--task", type=str, help="The name of the task", required=True)
+    args = parser.parse_args()
+
+    with open(args.input_file, "r") as file:
+        data = json.load(file)
+
+    processed_data = process_results(data, task=args.task)
+
+
+if __name__ == "__main__":
+    main()

From 65ab80c461b7c6a65dc4bab2e5aa3d46f18c82c0 Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel.com>
Date: Wed, 12 Feb 2025 20:03:56 +0000
Subject: [PATCH 015/112] [WIP] add the Pixtral model

---
 lmms_eval/models/__init__.py |   1 +
 lmms_eval/models/pixtral.py  | 318 +++++++++++++++++++++++++++++++++++
 requirements.txt             |   1 +
 3 files changed, 320 insertions(+)
 create mode 100644 lmms_eval/models/pixtral.py

diff --git a/lmms_eval/models/__init__.py b/lmms_eval/models/__init__.py
index a2082d34f..6627a0cee 100644
--- a/lmms_eval/models/__init__.py
+++ b/lmms_eval/models/__init__.py
@@ -57,6 +57,7 @@
     "llama_vision": "LlamaVision",
     "aria": "Aria",
     "pangea": "Pangea",
+    "pixtral": "Pixtral",
 }
 
 
diff --git a/lmms_eval/models/pixtral.py b/lmms_eval/models/pixtral.py
new file mode 100644
index 000000000..600ae04d2
--- /dev/null
+++ b/lmms_eval/models/pixtral.py
@@ -0,0 +1,318 @@
+import warnings
+from typing import List, Optional, Tuple, Union
+import os
+import torch
+from accelerate import Accelerator, DistributedType
+from accelerate.state import AcceleratorState
+from tqdm import tqdm
+from transformers import AutoProcessor, AutoModel
+from lmms_eval import utils
+from lmms_eval.api.instance import Instance
+from lmms_eval.api.model import lmms
+from lmms_eval.api.registry import register_model
+from mistral_inference.transformer import Transformer
+from mistral_inference.generate import generate
+from huggingface_hub import snapshot_download
+from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
+from mistral_common.protocol.instruct.messages import UserMessage, TextChunk, ImageURLChunk
+from mistral_common.protocol.instruct.request import ChatCompletionRequest
+
+warnings.filterwarnings("ignore")
+from loguru import logger as eval_logger
+
+DEFAULT_IMAGE_TOKEN = "<image>"
+
+@register_model("pixtral")
+class Pixtral(lmms):
+    """
+    Custom PIXTRAL model implementation using Hugging Face Transformers and remote code.
+
+    Example usage:
+
+    accelerate launch --num_processes=8 -m lmms_eval \
+        --model pixtral \
+        --model_args pretrained=mistralai/Pixtral-12B-Base-2409 \
+        --tasks mme \
+        --batch_size 1 \
+        --output_path ./logs/ \
+        --log_samples
+    """
+
+    def __init__(
+        self,
+        pretrained: str = "mistralai/Pixtral-12B-Base-2409",
+        model_path: str = None,
+        revision: str = "main",
+        device: str = "cuda",
+        dtype: Optional[Union[str, torch.dtype]] = "bfloat16",
+        batch_size: int = 1,
+        trust_remote_code: Optional[bool] = True,
+        #attn_implementation: Optional[str] = best_fit_attn_implementation,
+        add_system_prompt: Optional[str] = None,
+        tag: Optional[str] = None,
+        device_map: str = "",
+        use_cache: bool = True,
+        **kwargs,
+    ) -> None:
+        super().__init__()
+        assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
+
+        accelerator = Accelerator()
+        if accelerator.num_processes > 1:
+            self._device = torch.device(f"cuda:{accelerator.local_process_index}")
+            self.device_map = f"cuda:{accelerator.local_process_index}"
+        elif accelerator.num_processes == 1 and device_map == "auto":
+            self._device = torch.device(device)
+            self.device_map = device_map
+        else:
+            self._device = torch.device(f"cuda:{accelerator.local_process_index}")
+            self.device_map = f"cuda:{accelerator.local_process_index}"
+
+        if isinstance(dtype, str) and dtype != "auto":
+            dtype = getattr(torch, dtype)
+            
+        pretrained = self.download_model(pretrained, model_path)
+        self._model = Transformer.from_folder(
+            pretrained,
+            # revision=revision,
+            # trust_remote_code=trust_remote_code,
+            #attn_implementation=attn_implementation
+        )
+        # self._processor = AutoProcessor.from_pretrained(
+        #     pretrained,
+        #     revision=revision,
+        #     trust_remote_code=trust_remote_code
+        # )
+        self._tokenizer = MistralTokenizer.from_file(f"{pretrained}/tekken.json")
+        # self._config = self._model.config
+        self.batch_size_per_gpu = int(batch_size)
+        self.use_cache = use_cache
+        self.add_system_prompt = add_system_prompt
+        self.tag = tag
+        # Handle distributed setup
+        if accelerator.num_processes > 1:
+            assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED]
+            
+            if accelerator.distributed_type == DistributedType.DEEPSPEED:
+                kwargs = {
+                    "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
+                    "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
+                }
+                AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
+                eval_logger.info("Using DeepSpeed - ensure zero stage is set to 0 in accelerate config")
+
+            if accelerator.distributed_type in [DistributedType.FSDP, DistributedType.DEEPSPEED]:
+                self._model = accelerator.prepare(self.model)
+            else:
+                self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
+
+            self.accelerator = accelerator
+            if self.accelerator.is_local_main_process:
+                eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
+            self._rank = self.accelerator.local_process_index
+            self._world_size = self.accelerator.num_processes
+        else:
+            if device_map == "auto":
+                eval_logger.info("Using pipeline parallelism")
+            else:
+                eval_logger.info(f"Using single device: {self._device}")
+                self.model.to(self._device)
+            self._rank = 0
+            self._world_size = 1
+
+    def download_model(self, model_name: str, model_path: str):
+        
+        """
+        Download the model (given the model_name on hugging face, like 'mistralai/Pixtral-12B-Base-2409') to the specified model_path directory.
+        
+        Args:
+            model_name (str): The name of the model on Hugging Face (e.g., 'mistralai/Pixtral-12B-Base-2409')
+            model_path (str, optional): Directory to store the downloaded files. 
+                                    If None, uses the default HF cache directory.
+        
+        Returns:
+            str: Path to the directory containing the downloaded model files
+        """
+        try:
+            # If model_path is not provided, create one in the current directory
+            if model_path is None:
+                tmp_dir  = os.getenv("HF_HOME")
+                if tmp_dir is None:
+                    tmp_dir = "/mnt/data/cache/huggingface"
+                model_path = os.path.join(tmp_dir, model_name.replace("/", "__"))
+            
+            # Create model_path directory if it doesn't exist
+            os.makedirs(model_path, exist_ok=True)
+            
+            # Create a model-specific directory
+            model_dir = model_path
+            
+            # Check if model is already downloaded
+            if os.path.exists(model_dir) and len(os.listdir(model_dir)) > 0:
+                print(f"Model already exists at {model_dir}")
+                return model_dir
+                
+            print(f"Downloading model files from {model_name}...")
+            
+            # Download all files from the repo
+            local_dir = snapshot_download(
+                repo_id=model_name,
+                local_dir=model_dir,
+                local_dir_use_symlinks=False  # Download actual files, not symlinks
+            )
+            
+            print(f"Download completed. Files saved to: {local_dir}")
+            return local_dir
+            
+        except Exception as e:
+            print(f"Error downloading model files: {str(e)}")
+            return None
+        
+    
+    @property
+    def config(self):
+        return self._config
+
+    @property
+    def tokenizer(self):
+        return self._tokenizer
+
+    @property
+    def model(self):
+        if hasattr(self, "accelerator"):
+            return self.accelerator.unwrap_model(self._model)
+        else:
+            return self._model
+
+    @property
+    def eot_token_id(self):
+        return self.tokenizer.eos_token_id
+
+    @property
+    def max_length(self):
+        return self._max_length
+
+    @property
+    def batch_size(self):
+        return self.batch_size_per_gpu
+
+    @property
+    def device(self):
+        return self._device
+
+    @property
+    def rank(self):
+        return self._rank
+
+    @property
+    def world_size(self):
+        return self._world_size
+
+    def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
+        """Tokenize a string."""
+        breakpoint()
+        add_special_tokens = False if add_special_tokens is None else add_special_tokens
+        encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
+        if left_truncate_len:
+            encoding = encoding[-left_truncate_len:]
+        return encoding
+
+    def tok_decode(self, tokens):
+        """Decode a sequence of tokens to a string."""
+        return self.tokenizer.decode(tokens)
+
+    def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
+        """Not implemented for NVLM model."""
+        raise NotImplementedError("Loglikelihood is not implemented for NVLM model")
+
+    def flatten(self, input):
+        """Flatten a nested list."""
+        new_list = []
+        for i in input:
+            for j in i:
+                new_list.append(j)
+        return new_list
+
+    def generate_until(self, requests: List[Instance]) -> List[str]:
+        """Generate text based on the given requests."""
+        res = []
+
+        def _collate(x):
+            # the negative sign on len(toks) sorts descending - this has a few advantages:
+            # - time estimates will always be over not underestimates, which is more useful for planning
+            # - to know the size of a batch when going through the list, you know the first one is always the batch
+            #   padded context length. this is useful to simplify the batching logic and more importantly to make
+            #   automatic adaptive batches much much easier to implement
+            # - any OOMs will happen right away rather than near the end
+            toks = self.tok_encode(x[0])
+            return -len(toks), x[0]
+
+        def batch_requests(requests, batch_size):
+            for i in range(0, len(requests), batch_size):
+                yield [x.arguments for x in requests[i:i + batch_size]]
+
+        chunks = batch_requests(requests, self.batch_size)  
+        
+        num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
+        
+        pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
+        
+        for chunk in chunks:
+            contexts, all_gen_kwargs, doc_to_visuals, doc_id, tasks, splits = zip(*chunk)
+            visuals = [doc_to_visual(self.task_dict[task][split][ids]) for ids, task, split, doc_to_visual in zip(doc_id, tasks, splits, doc_to_visuals)]
+            
+            # Use the generation kwargs from the first request in the batch
+            gen_kwargs = all_gen_kwargs[0]
+            
+            # Set default generation parameters if not provided
+            if "max_new_tokens" not in gen_kwargs:
+                gen_kwargs["max_new_tokens"] = 1024
+            if "temperature" not in gen_kwargs:
+                gen_kwargs["temperature"] = 0
+            if "top_p" not in gen_kwargs:
+                gen_kwargs["top_p"] = None
+            if "num_beams" not in gen_kwargs:
+                gen_kwargs["num_beams"] = 1
+
+
+            # TODO: for now, images and text are passed seperatly to the processor
+            assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
+            context = contexts[0]
+            visual = visuals[0]
+            # TODO: handle multiple images / understand the `visuals` object
+            if isinstance(visual, list):
+                if len(visual) > 1:
+                    eval_logger.warning("More than one image is not supported for now... Using the first one")
+                visual = visual[0]
+            
+            if DEFAULT_IMAGE_TOKEN not in context:
+                context = f"{DEFAULT_IMAGE_TOKEN}\n{context}"
+            
+            if self.tag:
+                context = f"{self.tag} " + context
+            # create chat object and apply template
+            chat = [{"role": "user", "content": context}]
+            if self.add_system_prompt is not None:
+                chat.insert(0, {"role": "system", "content": self.add_system_prompt})
+
+            breakpoint()
+            completion_request = ChatCompletionRequest(messages=[UserMessage(content=[ImageURLChunk(image_url=visual), TextChunk(text=chat)])])
+
+            encoded = self.tokenizer.encode_chat_completion(completion_request)
+
+            images = encoded.images
+            tokens = encoded.tokens
+
+            out_tokens, _ = generate([tokens], self.model, images=[images], max_tokens=256, temperature=0.35, eos_id=self.tokenizer.instruct_tokenizer.tokenizer.eos_id)
+            result = self.tokenizer.decode(out_tokens[0])
+            res.append(result)
+            pbar.update(1)
+
+        # Reorder results back to original order
+        # res = re_ords.get_original(res)
+        pbar.close()
+        return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        """Not implemented for NVLM model."""
+        raise NotImplementedError("Multi-round generation is not implemented for NVLM model")
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 11e2127ac..c102c5140 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -181,3 +181,4 @@ yarl==1.18.3
 yt-dlp==2024.12.6
 zss==1.2.0
 zstandard==0.23.0
+mistral_inference

From 8d2f5a69f6ab851d8bd640a4837edfb1a9705278 Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel.com>
Date: Tue, 18 Feb 2025 15:48:43 +0000
Subject: [PATCH 016/112] [WIP] add the Pixtral Model

---
 lmms_eval/models/pixtral.py | 111 ++++++++++++++++++++++--------------
 1 file changed, 69 insertions(+), 42 deletions(-)

diff --git a/lmms_eval/models/pixtral.py b/lmms_eval/models/pixtral.py
index 600ae04d2..facab3d54 100644
--- a/lmms_eval/models/pixtral.py
+++ b/lmms_eval/models/pixtral.py
@@ -17,6 +17,31 @@
 from mistral_common.protocol.instruct.messages import UserMessage, TextChunk, ImageURLChunk
 from mistral_common.protocol.instruct.request import ChatCompletionRequest
 
+from transformers import LlavaForConditionalGeneration, AutoProcessor
+from PIL import Image
+from vllm import LLM
+from vllm.sampling_params import SamplingParams
+
+
+import PIL.Image
+import uuid
+from vllm import EngineArgs, LLMEngine
+from vllm import SamplingParams, TokensPrompt
+from vllm.multimodal import MultiModalDataBuiltins
+
+from mistral_common.protocol.instruct.messages import (
+    UserMessage,
+    TextChunk,
+    ImageURLChunk,
+    ImageChunk,
+)
+from PIL import Image
+from mistral_common.protocol.instruct.request import ChatCompletionRequest
+from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
+
+
+
+
 warnings.filterwarnings("ignore")
 from loguru import logger as eval_logger
 
@@ -42,12 +67,9 @@ def __init__(
         self,
         pretrained: str = "mistralai/Pixtral-12B-Base-2409",
         model_path: str = None,
-        revision: str = "main",
         device: str = "cuda",
         dtype: Optional[Union[str, torch.dtype]] = "bfloat16",
         batch_size: int = 1,
-        trust_remote_code: Optional[bool] = True,
-        #attn_implementation: Optional[str] = best_fit_attn_implementation,
         add_system_prompt: Optional[str] = None,
         tag: Optional[str] = None,
         device_map: str = "",
@@ -70,21 +92,19 @@ def __init__(
 
         if isinstance(dtype, str) and dtype != "auto":
             dtype = getattr(torch, dtype)
-            
-        pretrained = self.download_model(pretrained, model_path)
-        self._model = Transformer.from_folder(
-            pretrained,
-            # revision=revision,
-            # trust_remote_code=trust_remote_code,
-            #attn_implementation=attn_implementation
+
+        self.engine_args = EngineArgs(
+            model=pretrained,
+            tokenizer_mode="mistral",
+            enable_chunked_prefill=False,
+            limit_mm_per_prompt=dict(image=4),
+            max_num_batched_tokens=16384,
+            max_model_len=16384,
         )
-        # self._processor = AutoProcessor.from_pretrained(
-        #     pretrained,
-        #     revision=revision,
-        #     trust_remote_code=trust_remote_code
-        # )
-        self._tokenizer = MistralTokenizer.from_file(f"{pretrained}/tekken.json")
-        # self._config = self._model.config
+        self.sampling_params = SamplingParams(temperature=0.0, max_tokens=512)
+        self._engine = LLMEngine.from_engine_args(self.engine_args)
+        self._tokenizer = self._engine.tokenizer.tokenizer.mistral
+        
         self.batch_size_per_gpu = int(batch_size)
         self.use_cache = use_cache
         self.add_system_prompt = add_system_prompt
@@ -233,20 +253,34 @@ def flatten(self, input):
                 new_list.append(j)
         return new_list
 
+
+    def create_image_input(self, images, prompt):
+        # tokenize images and text
+        tokenized = self._tokenizer.encode_chat_completion(
+            ChatCompletionRequest(
+                messages=[
+                    UserMessage(
+                        content=[
+                            TextChunk(text=prompt),
+                        ] + [ImageChunk(image=img) for img in images]
+                    )
+                ],
+                model="pixtral",
+            )
+        )
+
+        engine_inputs = TokensPrompt(prompt_token_ids=tokenized.tokens)
+
+        mm_data = MultiModalDataBuiltins(image=images)
+        engine_inputs["multi_modal_data"] = mm_data
+
+        return engine_inputs
+
+
     def generate_until(self, requests: List[Instance]) -> List[str]:
         """Generate text based on the given requests."""
         res = []
 
-        def _collate(x):
-            # the negative sign on len(toks) sorts descending - this has a few advantages:
-            # - time estimates will always be over not underestimates, which is more useful for planning
-            # - to know the size of a batch when going through the list, you know the first one is always the batch
-            #   padded context length. this is useful to simplify the batching logic and more importantly to make
-            #   automatic adaptive batches much much easier to implement
-            # - any OOMs will happen right away rather than near the end
-            toks = self.tok_encode(x[0])
-            return -len(toks), x[0]
-
         def batch_requests(requests, batch_size):
             for i in range(0, len(requests), batch_size):
                 yield [x.arguments for x in requests[i:i + batch_size]]
@@ -287,29 +321,22 @@ def batch_requests(requests, batch_size):
             
             if DEFAULT_IMAGE_TOKEN not in context:
                 context = f"{DEFAULT_IMAGE_TOKEN}\n{context}"
-            
             if self.tag:
                 context = f"{self.tag} " + context
             # create chat object and apply template
             chat = [{"role": "user", "content": context}]
             if self.add_system_prompt is not None:
                 chat.insert(0, {"role": "system", "content": self.add_system_prompt})
-
-            breakpoint()
-            completion_request = ChatCompletionRequest(messages=[UserMessage(content=[ImageURLChunk(image_url=visual), TextChunk(text=chat)])])
-
-            encoded = self.tokenizer.encode_chat_completion(completion_request)
-
-            images = encoded.images
-            tokens = encoded.tokens
-
-            out_tokens, _ = generate([tokens], self.model, images=[images], max_tokens=256, temperature=0.35, eos_id=self.tokenizer.instruct_tokenizer.tokenizer.eos_id)
-            result = self.tokenizer.decode(out_tokens[0])
+            
+            try:
+                self._engine.add_request(uuid.uuid4().hex, self.create_image_input([visual], context), self.sampling_params)
+                out = self._engine.step()
+                result = out[0].outputs[0].text
+            except Exception as e:
+                eval_logger.error(f"Error {e} in generating")
+                result = ""
             res.append(result)
             pbar.update(1)
-
-        # Reorder results back to original order
-        # res = re_ords.get_original(res)
         pbar.close()
         return res
 

From 958cbb07f9e71e50bd32c219e787a96f086254c7 Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel.com>
Date: Wed, 19 Feb 2025 12:17:43 +0000
Subject: [PATCH 017/112] update the pixtral model to work with vLLM

---
 lmms_eval/models/pixtral.py | 109 ++++++++++++------------------------
 1 file changed, 36 insertions(+), 73 deletions(-)

diff --git a/lmms_eval/models/pixtral.py b/lmms_eval/models/pixtral.py
index facab3d54..1cef761fe 100644
--- a/lmms_eval/models/pixtral.py
+++ b/lmms_eval/models/pixtral.py
@@ -5,40 +5,17 @@
 from accelerate import Accelerator, DistributedType
 from accelerate.state import AcceleratorState
 from tqdm import tqdm
-from transformers import AutoProcessor, AutoModel
-from lmms_eval import utils
 from lmms_eval.api.instance import Instance
 from lmms_eval.api.model import lmms
 from lmms_eval.api.registry import register_model
-from mistral_inference.transformer import Transformer
-from mistral_inference.generate import generate
 from huggingface_hub import snapshot_download
-from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
-from mistral_common.protocol.instruct.messages import UserMessage, TextChunk, ImageURLChunk
-from mistral_common.protocol.instruct.request import ChatCompletionRequest
 
-from transformers import LlavaForConditionalGeneration, AutoProcessor
-from PIL import Image
-from vllm import LLM
 from vllm.sampling_params import SamplingParams
 
-
-import PIL.Image
-import uuid
-from vllm import EngineArgs, LLMEngine
-from vllm import SamplingParams, TokensPrompt
-from vllm.multimodal import MultiModalDataBuiltins
-
-from mistral_common.protocol.instruct.messages import (
-    UserMessage,
-    TextChunk,
-    ImageURLChunk,
-    ImageChunk,
-)
-from PIL import Image
-from mistral_common.protocol.instruct.request import ChatCompletionRequest
-from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
-
+import io
+import base64
+from vllm import LLM
+from vllm.sampling_params import SamplingParams
 
 
 
@@ -56,7 +33,7 @@ class Pixtral(lmms):
 
     accelerate launch --num_processes=8 -m lmms_eval \
         --model pixtral \
-        --model_args pretrained=mistralai/Pixtral-12B-Base-2409 \
+        --model_args pretrained=mistralai/mistralai/Pixtral-12B-2409 \
         --tasks mme \
         --batch_size 1 \
         --output_path ./logs/ \
@@ -65,8 +42,7 @@ class Pixtral(lmms):
 
     def __init__(
         self,
-        pretrained: str = "mistralai/Pixtral-12B-Base-2409",
-        model_path: str = None,
+        pretrained: str = "mistralai/Pixtral-12B-2409",
         device: str = "cuda",
         dtype: Optional[Union[str, torch.dtype]] = "bfloat16",
         batch_size: int = 1,
@@ -93,17 +69,8 @@ def __init__(
         if isinstance(dtype, str) and dtype != "auto":
             dtype = getattr(torch, dtype)
 
-        self.engine_args = EngineArgs(
-            model=pretrained,
-            tokenizer_mode="mistral",
-            enable_chunked_prefill=False,
-            limit_mm_per_prompt=dict(image=4),
-            max_num_batched_tokens=16384,
-            max_model_len=16384,
-        )
-        self.sampling_params = SamplingParams(temperature=0.0, max_tokens=512)
-        self._engine = LLMEngine.from_engine_args(self.engine_args)
-        self._tokenizer = self._engine.tokenizer.tokenizer.mistral
+        self._sampling_params = SamplingParams(max_tokens=8192)
+        self._model = LLM(model=pretrained, tokenizer_mode="mistral")
         
         self.batch_size_per_gpu = int(batch_size)
         self.use_cache = use_cache
@@ -254,28 +221,13 @@ def flatten(self, input):
         return new_list
 
 
-    def create_image_input(self, images, prompt):
-        # tokenize images and text
-        tokenized = self._tokenizer.encode_chat_completion(
-            ChatCompletionRequest(
-                messages=[
-                    UserMessage(
-                        content=[
-                            TextChunk(text=prompt),
-                        ] + [ImageChunk(image=img) for img in images]
-                    )
-                ],
-                model="pixtral",
-            )
-        )
-
-        engine_inputs = TokensPrompt(prompt_token_ids=tokenized.tokens)
-
-        mm_data = MultiModalDataBuiltins(image=images)
-        engine_inputs["multi_modal_data"] = mm_data
-
-        return engine_inputs
-
+    def get_image_url(self, image_bytes: bytes) -> str:
+        buffer = io.BytesIO()
+        image_bytes.save(buffer, format="JPEG")
+        img_bytes = buffer.getvalue()
+        base64_image = base64.b64encode(img_bytes).decode('utf-8')
+        data_url = f"data:image/jpeg;base64,{base64_image}"
+        return data_url
 
     def generate_until(self, requests: List[Instance]) -> List[str]:
         """Generate text based on the given requests."""
@@ -308,7 +260,6 @@ def batch_requests(requests, batch_size):
             if "num_beams" not in gen_kwargs:
                 gen_kwargs["num_beams"] = 1
 
-
             # TODO: for now, images and text are passed seperatly to the processor
             assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
             context = contexts[0]
@@ -318,26 +269,38 @@ def batch_requests(requests, batch_size):
                 if len(visual) > 1:
                     eval_logger.warning("More than one image is not supported for now... Using the first one")
                 visual = visual[0]
-            
-            if DEFAULT_IMAGE_TOKEN not in context:
-                context = f"{DEFAULT_IMAGE_TOKEN}\n{context}"
+
+            # vLLM does not work with bytes, so we need to convert it to a data url
+            image_url = self.get_image_url(visual)
+
+            # Pixtral expects inputs in a different format, and doesn't work with <image> tokens added in the middle of the prompt.
+            context = context.replace(DEFAULT_IMAGE_TOKEN, "")
+
             if self.tag:
                 context = f"{self.tag} " + context
-            # create chat object and apply template
-            chat = [{"role": "user", "content": context}]
+
+            # create chat object
+            message = [
+                {"role": "user",
+                 "content": [{"type": "text", "text": context}, {"type": "image_url", "image_url": {"url": image_url}}]
+                     }]
             if self.add_system_prompt is not None:
-                chat.insert(0, {"role": "system", "content": self.add_system_prompt})
+                message.insert(0, {"role": "system", "content": self.add_system_prompt})
+
             
             try:
-                self._engine.add_request(uuid.uuid4().hex, self.create_image_input([visual], context), self.sampling_params)
-                out = self._engine.step()
-                result = out[0].outputs[0].text
+                output = self._model.chat(message, sampling_params=self._sampling_params)
+                result = output[0].outputs[0].text
             except Exception as e:
                 eval_logger.error(f"Error {e} in generating")
                 result = ""
             res.append(result)
             pbar.update(1)
         pbar.close()
+        for rid, res in enumerate(res):
+            print(rid, res)
+            if rid >100:
+                break
         return res
 
     def generate_until_multi_round(self, requests) -> List[str]:

From a231572de21b771b4813b26e10caaa6dfb63866e Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel.com>
Date: Wed, 19 Feb 2025 12:41:51 +0000
Subject: [PATCH 018/112] Updated Pixtral model

---
 lmms_eval/models/pixtral.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/lmms_eval/models/pixtral.py b/lmms_eval/models/pixtral.py
index 1cef761fe..b72e7bc62 100644
--- a/lmms_eval/models/pixtral.py
+++ b/lmms_eval/models/pixtral.py
@@ -297,10 +297,6 @@ def batch_requests(requests, batch_size):
             res.append(result)
             pbar.update(1)
         pbar.close()
-        for rid, res in enumerate(res):
-            print(rid, res)
-            if rid >100:
-                break
         return res
 
     def generate_until_multi_round(self, requests) -> List[str]:

From 450e045e99a12adc953d623845edbe59cb0246d7 Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel.com>
Date: Wed, 19 Feb 2025 16:15:33 +0000
Subject: [PATCH 019/112] Updated Pixtral model

---
 lmms_eval/models/pixtral.py | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/lmms_eval/models/pixtral.py b/lmms_eval/models/pixtral.py
index b72e7bc62..d949d705a 100644
--- a/lmms_eval/models/pixtral.py
+++ b/lmms_eval/models/pixtral.py
@@ -69,8 +69,7 @@ def __init__(
         if isinstance(dtype, str) and dtype != "auto":
             dtype = getattr(torch, dtype)
 
-        self._sampling_params = SamplingParams(max_tokens=8192)
-        self._model = LLM(model=pretrained, tokenizer_mode="mistral")
+        self._model = LLM(model=pretrained, tokenizer_mode="mistral", gpu_memory_utilization=0.7, max_model_len=16384)
         
         self.batch_size_per_gpu = int(batch_size)
         self.use_cache = use_cache
@@ -197,7 +196,6 @@ def world_size(self):
 
     def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
         """Tokenize a string."""
-        breakpoint()
         add_special_tokens = False if add_special_tokens is None else add_special_tokens
         encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
         if left_truncate_len:
@@ -252,14 +250,11 @@ def batch_requests(requests, batch_size):
             
             # Set default generation parameters if not provided
             if "max_new_tokens" not in gen_kwargs:
-                gen_kwargs["max_new_tokens"] = 1024
+                gen_kwargs["max_new_tokens"] = 8192
             if "temperature" not in gen_kwargs:
                 gen_kwargs["temperature"] = 0
-            if "top_p" not in gen_kwargs:
-                gen_kwargs["top_p"] = None
-            if "num_beams" not in gen_kwargs:
-                gen_kwargs["num_beams"] = 1
 
+            self._sampling_params = SamplingParams(max_tokens=gen_kwargs["max_new_tokens"], temperature=gen_kwargs["temperature"])
             # TODO: for now, images and text are passed seperatly to the processor
             assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
             context = contexts[0]

From c8a6797165c4228f6194a38e6c75216f843b57ff Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel.com>
Date: Wed, 19 Feb 2025 18:01:41 +0000
Subject: [PATCH 020/112] added multi-image support

---
 lmms_eval/models/pixtral.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/lmms_eval/models/pixtral.py b/lmms_eval/models/pixtral.py
index d949d705a..7d0173d17 100644
--- a/lmms_eval/models/pixtral.py
+++ b/lmms_eval/models/pixtral.py
@@ -50,6 +50,7 @@ def __init__(
         tag: Optional[str] = None,
         device_map: str = "",
         use_cache: bool = True,
+        max_img_per_msg: int = 10,
         **kwargs,
     ) -> None:
         super().__init__()
@@ -69,7 +70,7 @@ def __init__(
         if isinstance(dtype, str) and dtype != "auto":
             dtype = getattr(torch, dtype)
 
-        self._model = LLM(model=pretrained, tokenizer_mode="mistral", gpu_memory_utilization=0.7, max_model_len=16384)
+        self._model = LLM(model=pretrained, tokenizer_mode="mistral", limit_mm_per_prompt={"image": max_img_per_msg}, gpu_memory_utilization=0.7, max_model_len=16384)
         
         self.batch_size_per_gpu = int(batch_size)
         self.use_cache = use_cache
@@ -240,7 +241,6 @@ def batch_requests(requests, batch_size):
         num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
         
         pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
-        
         for chunk in chunks:
             contexts, all_gen_kwargs, doc_to_visuals, doc_id, tasks, splits = zip(*chunk)
             visuals = [doc_to_visual(self.task_dict[task][split][ids]) for ids, task, split, doc_to_visual in zip(doc_id, tasks, splits, doc_to_visuals)]
@@ -259,14 +259,15 @@ def batch_requests(requests, batch_size):
             assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
             context = contexts[0]
             visual = visuals[0]
+            
             # TODO: handle multiple images / understand the `visuals` object
-            if isinstance(visual, list):
-                if len(visual) > 1:
-                    eval_logger.warning("More than one image is not supported for now... Using the first one")
-                visual = visual[0]
+            if not isinstance(visual, list):
+                visual = [visual]
 
             # vLLM does not work with bytes, so we need to convert it to a data url
-            image_url = self.get_image_url(visual)
+            image_urls = [self.get_image_url(v) for v in visual]
+            
+            # image_url = self.get_image_url(visual)
 
             # Pixtral expects inputs in a different format, and doesn't work with <image> tokens added in the middle of the prompt.
             context = context.replace(DEFAULT_IMAGE_TOKEN, "")
@@ -277,11 +278,10 @@ def batch_requests(requests, batch_size):
             # create chat object
             message = [
                 {"role": "user",
-                 "content": [{"type": "text", "text": context}, {"type": "image_url", "image_url": {"url": image_url}}]
+                 "content": [{"type": "text", "text": context}] + [{"type": "image_url", "image_url": {"url": image_url}} for image_url in image_urls]
                      }]
             if self.add_system_prompt is not None:
                 message.insert(0, {"role": "system", "content": self.add_system_prompt})
-
             
             try:
                 output = self._model.chat(message, sampling_params=self._sampling_params)

From 32a4c9fd26641665a5c519132f704c26b79d5f40 Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel.com>
Date: Fri, 28 Feb 2025 10:18:59 +0000
Subject: [PATCH 021/112] updated the parse results script

---
 scripts/parse_results.py | 361 +++++++++++++++++++++++++--------------
 1 file changed, 234 insertions(+), 127 deletions(-)

diff --git a/scripts/parse_results.py b/scripts/parse_results.py
index 2429a248c..70a73b116 100644
--- a/scripts/parse_results.py
+++ b/scripts/parse_results.py
@@ -1,148 +1,255 @@
 import argparse
 import json
-
-
-def get_results_m3exam(results):
-    score_en = results["m3exam_english"]["m3exam,none"]
-    results_multi = 0
-    results_multi = [
-        result["m3exam,none"]
-        for task, result in results.items()
-        if (task != "m3exam_english" and task != "m3exam")
-    ]
-    score_multi = sum(results_multi) / len(results_multi)
-    return score_en * 100, score_multi * 100
-
-
-def get_results_marvl(results):
-    score_en = results["nlvr2"]["exact_match,none"]
-    results_multi = 0
-    results_multi = [
-        result["exact_match,none"]
-        for task, result in results.items()
-        if (task != "marvl" and task != "nlvr2")
-    ]
-    score_multi = sum(results_multi) / len(results_multi)
-    return score_en * 100, score_multi * 100
-
-
-def get_results_maxm(results):
-    score_en = results["maxm_en"]["relaxed_accuracy,none"]
-    results_multi = 0
-    results_multi = [
-        result["relaxed_accuracy,none"]
-        for task, result in results.items()
-        if (task != "maxm" and task != "maxm_en")
-    ]
-    score_multi = sum(results_multi) / len(results_multi)
-    return score_en, score_multi
-
-
-def get_results_xgqa(results):
-    score_en = results["xgqa_en"]["exact_match,none"]
-    results_multi = 0
-    results_multi = [
-        result["exact_match,none"]
-        for task, result in results.items()
-        if (task != "xgqa" and task != "xgqa_en")
-    ]
-    score_multi = sum(results_multi) / len(results_multi)
-    return score_en * 100, score_multi * 100
-
-
-def get_results_xm100(results):
-    score_en = results["xm100_en"]["xm100_ROUGE_L,none"]
-    results_multi = 0
-    results_multi = [
-        result["xm100_ROUGE_L,none"]
-        for task, result in results.items()
-        if (task != "xm100" and task != "xm100_en")
-    ]
-    score_multi = sum(results_multi) / len(results_multi)
-    return score_en * 100, score_multi * 100
-
-
-def get_results_xmmmu(results):
-    score_en = results["mmmu_English_val"]["mmmu_acc,none"]
-    results_multi = 0
-    results_multi = [
-        result["mmmu_acc,none"]
-        for task, result in results.items()
-        if (task != "xmmmu" and task != "mmmu_English_val")
-    ]
-    score_multi = sum(results_multi) / len(results_multi)
-    return score_en * 100, score_multi * 100
-
-
-def get_results_multilingual_llava_bench(results):
-    score_en = results["multilingual_llava_bench_english"]["gpt_eval_llava_all,none"]
-    results_multi = 0
-    results_multi = [
-        result["gpt_eval_llava_all,none"]
-        for task, result in results.items()
-        if (
-            task != "multilingual_llava_bench"
-            and task != "multilingual_llava_bench_english"
-        )
-    ]
-    score_multi = sum(results_multi) / len(results_multi)
-    return score_en, score_multi
-
-
-def get_results_ai2d(results):
-    score_en = results["ai2d"]["exact_match,flexible-extract"]
-    return score_en * 100, None
-
-
-def get_results_mmmu(results):
-    score_en = results["mmmu_val_group_img"]["mmmu_acc,none"]
-    return score_en * 100, None
-
-
-def get_results_textvqa(results):
-    score_en = results["textvqa_val"]["exact_match,none"]
-    return score_en * 100, None
-
-def get_results_scienceqa(results):
-    score_en = results["scienceqa"]["exact_match,none"]
-    return score_en * 100, None
-
-def process_results(data, task):
+import glob
+import os
+from tabulate import tabulate
+
+def get_results_m3exam(results, task_name, model_name):
+    rows = []
+    scores = {}
+    scores["en"] = float(results["m3exam_english"]["m3exam,none"]) * 100
+    scores["it"] = float(results["m3exam_italian"]["m3exam,none"]) * 100
+    scores["pt"] = float(results["m3exam_portuguese"]["m3exam,none"]) * 100
+    scores["zh"] = float(results["m3exam_chinese"]["m3exam,none"]) * 100
+    
+    for kk, vv in scores.items():
+        rows.append({
+            "model": model_name,
+            "task": task_name,
+            "language": kk,
+            "score": vv
+        })
+    return rows
+
+def get_results_chartqa(results, task_name, model_name):
+    rows = []
+    scores = {}
+    scores["en"] = float(results["chartqa"]["relaxed_overall,none"]) * 100
+    
+    for kk, vv in scores.items():
+        rows.append({
+            "model": model_name,
+            "task": task_name,
+            "language": kk,
+            "score": vv
+        })
+    return rows
+
+"""
+We use Exact Match Accuracy for this task.
+This is based on the information provided in the origial paper: 
+MaXM: Towards Multilingual Visual Question Answering (https://arxiv.org/pdf/2209.05401)
+Section 5.1:
+Evaluation Metrics. We use Exact Match Accu-
+racy as the main evaluation measure for MaXM,
+following previous work on VQA
+"""
+def get_results_maxm(results, task_name, model_name):
+    rows = []
+    scores = {}
+    scores["en"] = float(results["maxm_en"]["relaxed_accuracy,none"])
+    scores["fr"] = float(results["maxm_fr"]["relaxed_accuracy,none"])
+    scores["zh"] = float(results["maxm_zh"]["relaxed_accuracy,none"])
+    
+    for kk, vv in scores.items():
+        rows.append({
+            "model": model_name,
+            "task": task_name,
+            "language": kk,
+            "score": vv
+        })
+    return rows
+
+def get_results_mme(results, task_name, model_name):
+    rows = []
+    scores = {}
+    scores["en"] = f"{round(results['mme']['mme_cognition_score,none'],2)}/{round(results['mme']['mme_perception_score,none'],2)}"
+    
+    for kk, vv in scores.items():
+        rows.append({
+            "model": model_name,
+            "task": task_name,
+            "language": kk,
+            "score": vv
+        })
+    return rows
+
+def get_results_mmmu(results, task_name, model_name):
+    rows = []
+    scores = {}
+    scores["en"] = float(results["mmmu_val"]["mmmu_acc,none"]) * 100
+    
+    for kk, vv in scores.items():
+        rows.append({
+            "model": model_name,
+            "task": task_name,
+            "language": kk,
+            "score": vv
+        })
+    return rows
+
+def get_results_ocrbench(results, task_name, model_name):
+    rows = []
+    scores = {}
+    scores["en"] = float(results["ocrbench"]["ocrbench_accuracy,none"]) * 100
+    
+    for kk, vv in scores.items():
+        rows.append({
+            "model": model_name,
+            "task": task_name,
+            "language": kk,
+            "score": vv
+        })
+    return rows
+
+def get_results_scienceqa(results, task_name, model_name):
+    rows = []
+    scores = {}
+    scores["en"] = float(results["scienceqa"]["exact_match,none"]) * 100
+    
+    for kk, vv in scores.items():
+        rows.append({
+            "model": model_name,
+            "task": task_name,
+            "language": kk,
+            "score": vv
+        })
+    return rows
+
+def get_results_textvqa(results, task_name, model_name):
+    rows = []
+    scores = {}
+    scores["en"] = float(results["textvqa_val"]["exact_match,none"]) * 100
+    
+    for kk, vv in scores.items():
+        rows.append({
+            "model": model_name,
+            "task": task_name,
+            "language": kk,
+            "score": vv
+        })
+    return rows
+
+def get_results_xgqa(results, task_name, model_name):
+    rows = []
+    scores = {}
+    scores["de"] = float(results["xgqa_de"]["exact_match,none"]) * 100
+    scores["en"] = float(results["xgqa_en"]["exact_match,none"]) * 100
+    scores["ko"] = float(results["xgqa_ko"]["exact_match,none"]) * 100
+    scores["pt"] = float(results["xgqa_pt"]["exact_match,none"]) * 100
+    scores["ru"] = float(results["xgqa_ru"]["exact_match,none"]) * 100
+    scores["zh"] = float(results["xgqa_zh"]["exact_match,none"]) * 100
+    
+    for kk, vv in scores.items():
+        rows.append({
+            "model": model_name,
+            "task": task_name,
+            "language": kk,
+            "score": vv
+        })
+    return rows
+
+# def get_results_xm100(results, task_name, model_name):
+#     rows = []
+#     scores = {}
+#     scores["de"] = float(results["xm100_de"]["xm100_CIDEr,none"]) * 100
+#     scores["en"] = float(results["xm100_en"]["xm100_CIDEr,none"]) * 100
+#     scores["es"] = float(results["xm100_es"]["xm100_CIDEr,none"]) * 100
+#     scores["fr"] = float(results["xm100_fr"]["xm100_CIDEr,none"]) * 100
+#     scores["it"] = float(results["xm100_it"]["xm100_CIDEr,none"]) * 100
+#     scores["ko"] = float(results["xm100_ko"]["xm100_CIDEr,none"]) * 100
+#     scores["nl"] = float(results["xm100_nl"]["xm100_CIDEr,none"]) * 100
+#     scores["pt"] = float(results["xm100_pt"]["xm100_CIDEr,none"]) * 100
+#     scores["ru"] = float(results["xm100_ru"]["xm100_CIDEr,none"]) * 100
+#     scores["zh"] = float(results["xm100_zh"]["xm100_CIDEr,none"]) * 100
+    
+#     for kk, vv in scores.items():
+#         rows.append({
+#             "model": model_name,
+#             "task": task_name,
+#             "language": kk,
+#             "score": vv
+#         })
+#     return rows
+
+def get_results_xmmmu(results, task_name, model_name):
+    rows = []
+    scores = {}
+    scores["en"] = float(results["mmmu_English_val"]["mmmu_acc,none"]) * 100
+    scores["fr"] = float(results["mmmu_French_val"]["mmmu_acc,none"]) * 100
+    scores["pt"] = float(results["mmmu_Portuguese_val"]["mmmu_acc,none"]) * 100
+    
+    for kk, vv in scores.items():
+        rows.append({
+            "model": model_name,
+            "task": task_name,
+            "language": kk,
+            "score": vv
+        })
+    return rows
+
+
+def get_results_ai2d(results, task_name, model_name):
+    rows = []
+    scores = {}
+    scores["en"] = float(results["ai2d"]["exact_match,flexible-extract"]) * 100
+    
+    for kk, vv in scores.items():
+        rows.append({
+            "model": model_name,
+            "task": task_name,
+            "language": kk,
+            "score": vv
+        })
+    return rows
+
+def process_results(data, task_name, model_name):
     results = data["results"]
-    score_en, score_multi = result_functions[task](results)
-    print(f"============== Results for {task} ==============")
-    if score_multi:
-        print(f"{score_en:.2f}\t{score_multi:.2f}")
-    else:
-        print(f"{score_en:.2f}")
+    results = result_functions[task_name](results, task_name, model_name)
+    return results
 
 
 result_functions = {
-    "ai2d": get_results_ai2d,
-    "mmmu": get_results_mmmu,
-    "textvqa": get_results_textvqa,
     "m3exam": get_results_m3exam,
-    "marvl": get_results_marvl,
+    "chartqa": get_results_chartqa,
     "maxm": get_results_maxm,
+    "mme": get_results_mme,
+    "mmmu": get_results_mmmu,
+    "ocrbench": get_results_ocrbench,
+    "scienceqa": get_results_scienceqa,
+    "textvqa": get_results_textvqa,
     "xgqa": get_results_xgqa,
-    "xm100": get_results_xm100,
+    # "xm100": get_results_xm100,
     "xmmmu": get_results_xmmmu,
-    "mllava_bench": get_results_multilingual_llava_bench,
-    "scienceqa": get_results_scienceqa,
+    "ai2d": get_results_ai2d,
 }
 
 
 def main():
     parser = argparse.ArgumentParser(description="Process a JSON file.")
-    parser.add_argument("--input_file", type=str, help="Path to the input JSON file")
+    parser.add_argument("--input-dir", type=str, help="Path to the input JSON file")
     parser.add_argument("--task", type=str, help="The name of the task", required=True)
+    parser.add_argument("--model", type=str, help="The name of the model", required=True)
     args = parser.parse_args()
-
-    with open(args.input_file, "r") as file:
+    
+    model_name = args.model.replace("/", "__")
+    task = args.task
+    
+    
+
+    input_dir = os.path.join(args.input_dir, task, model_name)
+    input_files = glob.glob(os.path.join(input_dir, "*results.json"))
+    if not input_files:
+        print(f"No files found in directory {args.input_dir} with pattern *results.json")
+    assert len(input_files) == 1, f"Found more than one file in {args.input_dir} with pattern *results.json"
+    input_file = input_files[0]
+
+    
+    
+    with open(input_file, "r") as file:
         data = json.load(file)
-
-    processed_data = process_results(data, task=args.task)
-
+    processed_data = process_results(data, task_name=args.task, model_name=model_name)
+    print(tabulate(processed_data, headers="keys"))
 
 if __name__ == "__main__":
     main()

From 1a466f356e2921c9ff385f764a26c3653d16a78f Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Mon, 3 Mar 2025 19:12:27 +0000
Subject: [PATCH 022/112] added cc-ocr

---
 lmms_eval/tasks/cc_ocr/__init__.py            |  0
 .../tasks/cc_ocr/cc-ocr-multi-lan-de.yaml     | 39 ++++++++
 .../tasks/cc_ocr/cc-ocr-multi-lan-es.yaml     | 39 ++++++++
 .../tasks/cc_ocr/cc-ocr-multi-lan-fr.yaml     | 39 ++++++++
 .../tasks/cc_ocr/cc-ocr-multi-lan-it.yaml     | 39 ++++++++
 .../tasks/cc_ocr/cc-ocr-multi-lan-ko.yaml     | 39 ++++++++
 .../tasks/cc_ocr/cc-ocr-multi-lan-pt.yaml     | 39 ++++++++
 .../tasks/cc_ocr/cc-ocr-multi-lan-ru.yaml     | 39 ++++++++
 lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan.yaml  |  8 ++
 lmms_eval/tasks/cc_ocr/cc_ocr_eval/utils.py   | 70 +++++++++++++++
 lmms_eval/tasks/cc_ocr/utils.py               | 88 +++++++++++++++++++
 11 files changed, 439 insertions(+)
 create mode 100644 lmms_eval/tasks/cc_ocr/__init__.py
 create mode 100644 lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml
 create mode 100644 lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-es.yaml
 create mode 100644 lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-fr.yaml
 create mode 100644 lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-it.yaml
 create mode 100644 lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ko.yaml
 create mode 100644 lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-pt.yaml
 create mode 100644 lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ru.yaml
 create mode 100644 lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan.yaml
 create mode 100644 lmms_eval/tasks/cc_ocr/cc_ocr_eval/utils.py
 create mode 100644 lmms_eval/tasks/cc_ocr/utils.py

diff --git a/lmms_eval/tasks/cc_ocr/__init__.py b/lmms_eval/tasks/cc_ocr/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml
new file mode 100644
index 000000000..c9113bb2a
--- /dev/null
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml
@@ -0,0 +1,39 @@
+dataset_path: wulipc/CC-OCR
+task: cc-ocr-multi-lan-de
+
+
+dataset_name: multi_lan_ocr 
+test_split: test
+
+dataset_kwargs:
+  token: True
+  data_files:
+    test: [ "multi_lan_ocr/German/*.tsv", 
+    #"multi_lan_ocr/German/*.tsv", 
+    # "multi_lan_ocr/Italian/*.tsv" ,
+    # "multi_lan_ocr/Korean/*.tsv",
+     #"multi_lan_ocr/Portuguese/*.tsv",
+     # "multi_lan_ocr/Russian/*.tsv",
+     # "multi_lan_ocr/Spanish/*.tsv"
+     ] 
+
+process_docs:  !function utils.ccocr_process_docs
+doc_to_visual: !function utils.ccocr_doc_to_visual
+doc_to_text: !function utils.ccocr_doc_to_text
+process_results: !function utils.ccocr_process_results
+doc_to_target: "answer"
+
+output_type: generate_until
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  do_sample: False
+
+
+metric_list:
+  - metric: ocr_results
+    aggregation: !function utils.ccocr_multi_lan_ocr_aggregate_results
+    higher_is_better: true
+
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-es.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-es.yaml
new file mode 100644
index 000000000..4b248cc86
--- /dev/null
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-es.yaml
@@ -0,0 +1,39 @@
+dataset_path: wulipc/CC-OCR
+task: cc-ocr-multi-lan-es
+
+
+dataset_name: multi_lan_ocr 
+test_split: test
+
+dataset_kwargs:
+  token: True
+  data_files:
+    test: [ #"multi_lan_ocr/German/*.tsv", 
+    # "multi_lan_ocr/French/*.tsv", 
+    # "multi_lan_ocr/Italian/*.tsv" ,
+    # "multi_lan_ocr/Korean/*.tsv",
+    #  "multi_lan_ocr/Portuguese/*.tsv",
+    #  "multi_lan_ocr/Russian/*.tsv",
+     "multi_lan_ocr/Spanish/*.tsv"
+     ] 
+
+process_docs:  !function utils.ccocr_process_docs
+doc_to_visual: !function utils.ccocr_doc_to_visual
+doc_to_text: !function utils.ccocr_doc_to_text
+process_results: !function utils.ccocr_process_results
+doc_to_target: "answer"
+
+output_type: generate_until
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  do_sample: False
+
+
+metric_list:
+  - metric: ocr_results
+    aggregation: !function utils.ccocr_multi_lan_ocr_aggregate_results
+    higher_is_better: true
+
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-fr.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-fr.yaml
new file mode 100644
index 000000000..1c4b60b5e
--- /dev/null
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-fr.yaml
@@ -0,0 +1,39 @@
+dataset_path: wulipc/CC-OCR
+task: cc-ocr-multi-lan-fr
+
+
+dataset_name: multi_lan_ocr 
+test_split: test
+
+dataset_kwargs:
+  token: True
+  data_files:
+    test: [ #"multi_lan_ocr/German/*.tsv", 
+    "multi_lan_ocr/French/*.tsv", 
+    # "multi_lan_ocr/Italian/*.tsv" ,
+    # "multi_lan_ocr/Korean/*.tsv",
+     #"multi_lan_ocr/Portuguese/*.tsv",
+     # "multi_lan_ocr/Russian/*.tsv",
+     # "multi_lan_ocr/Spanish/*.tsv"
+     ] 
+
+process_docs:  !function utils.ccocr_process_docs
+doc_to_visual: !function utils.ccocr_doc_to_visual
+doc_to_text: !function utils.ccocr_doc_to_text
+process_results: !function utils.ccocr_process_results
+doc_to_target: "answer"
+
+output_type: generate_until
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  do_sample: False
+
+
+metric_list:
+  - metric: ocr_results
+    aggregation: !function utils.ccocr_multi_lan_ocr_aggregate_results
+    higher_is_better: true
+
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-it.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-it.yaml
new file mode 100644
index 000000000..c3354d3aa
--- /dev/null
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-it.yaml
@@ -0,0 +1,39 @@
+dataset_path: wulipc/CC-OCR
+task: cc-ocr-multi-lan-it
+
+
+dataset_name: multi_lan_ocr 
+test_split: test
+
+dataset_kwargs:
+  token: True
+  data_files:
+    test: [ #"multi_lan_ocr/German/*.tsv", 
+    # "multi_lan_ocr/French/*.tsv", 
+    "multi_lan_ocr/Italian/*.tsv" ,
+    # "multi_lan_ocr/Korean/*.tsv",
+    #  "multi_lan_ocr/Portuguese/*.tsv",
+     # "multi_lan_ocr/Russian/*.tsv",
+     # "multi_lan_ocr/Spanish/*.tsv"
+     ] 
+
+process_docs:  !function utils.ccocr_process_docs
+doc_to_visual: !function utils.ccocr_doc_to_visual
+doc_to_text: !function utils.ccocr_doc_to_text
+process_results: !function utils.ccocr_process_results
+doc_to_target: "answer"
+
+output_type: generate_until
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  do_sample: False
+
+
+metric_list:
+  - metric: ocr_results
+    aggregation: !function utils.ccocr_multi_lan_ocr_aggregate_results
+    higher_is_better: true
+
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ko.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ko.yaml
new file mode 100644
index 000000000..2e43d68b1
--- /dev/null
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ko.yaml
@@ -0,0 +1,39 @@
+dataset_path: wulipc/CC-OCR
+task: cc-ocr-multi-lan-ko
+
+
+dataset_name: multi_lan_ocr 
+test_split: test
+
+dataset_kwargs:
+  token: True
+  data_files:
+    test: [ #"multi_lan_ocr/German/*.tsv", 
+    # "multi_lan_ocr/French/*.tsv", 
+    # "multi_lan_ocr/Italian/*.tsv" ,
+    "multi_lan_ocr/Korean/*.tsv",
+    #  "multi_lan_ocr/Portuguese/*.tsv",
+    #  "multi_lan_ocr/Russian/*.tsv",
+     # "multi_lan_ocr/Spanish/*.tsv"
+     ] 
+
+process_docs:  !function utils.ccocr_process_docs
+doc_to_visual: !function utils.ccocr_doc_to_visual
+doc_to_text: !function utils.ccocr_doc_to_text
+process_results: !function utils.ccocr_process_results
+doc_to_target: "answer"
+
+output_type: generate_until
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  do_sample: False
+
+
+metric_list:
+  - metric: ocr_results
+    aggregation: !function utils.ccocr_multi_lan_ocr_aggregate_results
+    higher_is_better: true
+
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-pt.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-pt.yaml
new file mode 100644
index 000000000..9fc4c3577
--- /dev/null
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-pt.yaml
@@ -0,0 +1,39 @@
+dataset_path: wulipc/CC-OCR
+task: cc-ocr-multi-lan-pt
+
+
+dataset_name: multi_lan_ocr 
+test_split: test
+
+dataset_kwargs:
+  token: True
+  data_files:
+    test: [ #"multi_lan_ocr/German/*.tsv", 
+    # "multi_lan_ocr/French/*.tsv", 
+    # "multi_lan_ocr/Italian/*.tsv" ,
+    # "multi_lan_ocr/Korean/*.tsv",
+     "multi_lan_ocr/Portuguese/*.tsv",
+     # "multi_lan_ocr/Russian/*.tsv",
+     # "multi_lan_ocr/Spanish/*.tsv"
+     ] 
+
+process_docs:  !function utils.ccocr_process_docs
+doc_to_visual: !function utils.ccocr_doc_to_visual
+doc_to_text: !function utils.ccocr_doc_to_text
+process_results: !function utils.ccocr_process_results
+doc_to_target: "answer"
+
+output_type: generate_until
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  do_sample: False
+
+
+metric_list:
+  - metric: ocr_results
+    aggregation: !function utils.ccocr_multi_lan_ocr_aggregate_results
+    higher_is_better: true
+
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ru.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ru.yaml
new file mode 100644
index 000000000..6435c9198
--- /dev/null
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ru.yaml
@@ -0,0 +1,39 @@
+dataset_path: wulipc/CC-OCR
+task: cc-ocr-multi-lan-ru
+
+
+dataset_name: multi_lan_ocr 
+test_split: test
+
+dataset_kwargs:
+  token: True
+  data_files:
+    test: [ #"multi_lan_ocr/German/*.tsv", 
+    # "multi_lan_ocr/French/*.tsv", 
+    # "multi_lan_ocr/Italian/*.tsv" ,
+    # "multi_lan_ocr/Korean/*.tsv",
+    #  "multi_lan_ocr/Portuguese/*.tsv",
+     "multi_lan_ocr/Russian/*.tsv",
+     # "multi_lan_ocr/Spanish/*.tsv"
+     ] 
+
+process_docs:  !function utils.ccocr_process_docs
+doc_to_visual: !function utils.ccocr_doc_to_visual
+doc_to_text: !function utils.ccocr_doc_to_text
+process_results: !function utils.ccocr_process_results
+doc_to_target: "answer"
+
+output_type: generate_until
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  do_sample: False
+
+
+metric_list:
+  - metric: ocr_results
+    aggregation: !function utils.ccocr_multi_lan_ocr_aggregate_results
+    higher_is_better: true
+
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan.yaml
new file mode 100644
index 000000000..1a63d5f56
--- /dev/null
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan.yaml
@@ -0,0 +1,8 @@
+group: cc-ocr-multi-lan
+task:
+  - cc-ocr-multi-lan-de
+  - cc-ocr-multi-lan-fr
+  # - cc-ocr-multi-lan-it
+  # - cc-ocr-multi-lan-ru
+  # - cc-ocr-multi-lan-es
+  - cc-ocr-multi-lan-ko
\ No newline at end of file
diff --git a/lmms_eval/tasks/cc_ocr/cc_ocr_eval/utils.py b/lmms_eval/tasks/cc_ocr/cc_ocr_eval/utils.py
new file mode 100644
index 000000000..88d0c149d
--- /dev/null
+++ b/lmms_eval/tasks/cc_ocr/cc_ocr_eval/utils.py
@@ -0,0 +1,70 @@
+"""
+Code adapted from:
+https://github.com/AlibabaResearch/AdvancedLiterateMachinery/blob/main/Benchmarks/CC-OCR/evaluation/evaluator.py
+"""
+import re
+from collections import Counter
+
+def token_normalize(token_text, is_lower=False, is_alphanum_only=False):
+    """
+    """
+    if is_lower:
+        token_text = token_text.lower()
+    if is_alphanum_only:
+        token_text = re.sub('[^A-Za-z0-9]+', '', token_text)
+    return token_text
+
+
+def text_normalize_and_tokenize(text, is_keep_blank=True, is_lower=True, is_alphanum_only=False):
+    text = text.replace("\t", " ").replace("\n", " ").replace("###", "").replace("***", "")
+    text = re.sub(r'\s+', ' ', text)
+    if not is_keep_blank:
+        text = text.replace(" ", "")
+    text_tokens = text.split(" ") if is_keep_blank else list(text)
+    text_token_normalized = [token_normalize(t, is_lower, is_alphanum_only) for t in text_tokens]
+    text_token_normalized = [x for x in text_token_normalized if len(x) > 0]
+    return text_token_normalized
+
+
+def evaluate_single_sample(gts, preds):
+    right_num = 0
+    gt_counter_info = dict(Counter(gts))
+    pdt_counter_info = dict(Counter(preds))
+    for gt_token, gt_count in gt_counter_info.items():
+        pred_count = pdt_counter_info.get(gt_token, 0)
+        right_num += min(gt_count, pred_count)
+    return right_num
+
+def calculate_metrics(predictions, groundtruths, is_verbose=False):
+    """
+    """
+    macro_recall_list, macro_precision_list, macro_f1_list = [], [], []
+    total_gt_num, total_pred_num, total_right_num = 0, 0, 0
+    for pred_tokenized, gt_tokenized in zip(predictions,groundtruths):
+        right_num = evaluate_single_sample(gt_tokenized, pred_tokenized)
+        total_right_num += right_num
+        total_gt_num += len(gt_tokenized)
+        total_pred_num += len(pred_tokenized)
+
+        macro_recall = right_num / (len(gt_tokenized) + 1e-9)
+        macro_precision = right_num / (len(pred_tokenized) + 1e-9)
+        macro_f1 = 2 * macro_recall * macro_precision / (macro_recall + macro_precision + 1e-9)
+        macro_recall_list.append(macro_recall)
+        macro_precision_list.append(macro_precision)
+        macro_f1_list.append(macro_f1)
+
+    # marco
+    final_macro_recall = sum(macro_recall_list) / (len(macro_recall_list) + 1e-9)
+    final_macro_precision = sum(macro_precision_list) / (len(macro_precision_list) + 1e-9)
+    final_macro_f1 = sum(macro_f1_list) / (len(macro_f1_list) + 1e-9)
+
+    # micro
+    recall_acc = total_right_num / (total_gt_num + 1e-9)
+    preci_acc = total_right_num / (total_pred_num + 1e-9)
+    hmean = 2 * recall_acc * preci_acc / (recall_acc + preci_acc + 1e-9)
+    vbs_eval_result = {
+        'macro_recall': final_macro_recall, 'macro_precision': final_macro_precision, 'macro_f1_score': final_macro_f1,
+        'micro_recall': recall_acc, 'micro_precision': preci_acc, 'mirco_f1_score': hmean
+    }
+    eval_result = vbs_eval_result if is_verbose else {'macro_f1_score': final_macro_f1, 'mirco_f1_score': hmean}
+    return eval_result
\ No newline at end of file
diff --git a/lmms_eval/tasks/cc_ocr/utils.py b/lmms_eval/tasks/cc_ocr/utils.py
new file mode 100644
index 000000000..bf2302326
--- /dev/null
+++ b/lmms_eval/tasks/cc_ocr/utils.py
@@ -0,0 +1,88 @@
+
+import base64
+from PIL import Image
+from io import BytesIO
+import numpy as np
+
+from loguru import logger
+
+from lmms_eval.tasks.cc_ocr.cc_ocr_eval.utils import text_normalize_and_tokenize, calculate_metrics
+
+def base64_to_bytes(base64_string):
+    # Remove the header if it exists (e.g., "data:image/jpeg;base64,")
+    if "base64," in base64_string:
+        base64_string = base64_string.split("base64,")[1]
+    # Decode base64 string to bytes
+    img_bytes = base64.b64decode(base64_string)
+    return img_bytes
+
+
+def ccocr_process_docs(docs):
+    """
+    Process documents by converting base64 images to PIL Images
+    
+    Args:
+        docs: Dataset object containing documents with base64 encoded images
+    Returns:
+        Dataset with converted PIL images
+    """
+    logger.info(f"Converting base64 images to PIL Images...")
+    # Process images in place
+    docs = docs.map(
+        lambda doc: {
+            'image': base64_to_bytes(doc['image'])
+        }
+    )
+    return docs
+
+def ccocr_doc_to_visual(doc):
+    image = Image.open(BytesIO(doc["image"]))
+    image = image.convert('RGB')
+    return [image]
+
+
+def ccocr_doc_to_text(doc, ):
+    question = doc["question"]
+    return f"{question}"
+
+
+def ccocr_process_results(doc, results):
+    pred = results[0]
+    answer = doc["answer"]
+    return {"ocr_results": {"index": doc["index"], "image_name": doc["image_name"],"question_type": doc["question"], "prediction": pred, "ground_truth": answer} }
+
+
+
+def ccocr_multi_lan_ocr_aggregate_results(results):
+    """
+    The multi-language-ocr evaluation according to the paper (https://arxiv.org/pdf/2412.02210) is done in different ways depending on the language.
+    For "Arabic", "Japanese", "Korean", and "zh" the evaluation procedure is using characters as basic units.
+    Otherwise, the evaluation procedure is using words as basic units.
+    This function implements the multi-language-ocr evaluation procedure for languages that are not in ["Arabic", "Japanese", "Korean", "zh"].
+    The code is taken from https://github.com/AlibabaResearch/AdvancedLiterateMachinery/blob/main/Benchmarks/CC-OCR/evaluation/evaluator/ocr_evaluator.py.
+    """
+    # The code below is implemented according to https://github.com/AlibabaResearch/AdvancedLiterateMachinery/blob/main/Benchmarks/CC-OCR/evaluation/evaluator/ocr_evaluator.py
+
+    logger.info("Aggregating results for languages that are not in ['Arabic', 'Japanese', 'Korean', 'zh'].")
+
+    non_basic_languages = ["Arabic", "Japanese", "Korean", "Chinese"]
+    pdt_tokenized, gt_tokenized = [], []
+    for res  in results:
+        index = res["index"]
+        image_name = res["image_name"]
+        pred = res["prediction"]
+        gt = res["ground_truth"]
+        
+        is_word_level, is_lower, is_alphanum_only = True, True, False
+        language = image_name.split("_")[0]
+        if language in non_basic_languages:
+            is_word_level = False
+        
+        
+        pdt_token_list = text_normalize_and_tokenize(pred.strip(), is_word_level, is_lower, is_alphanum_only)
+        gt_token_list = text_normalize_and_tokenize(gt.strip(), is_word_level, is_lower, is_alphanum_only)
+        pdt_tokenized.append(pdt_token_list)
+        gt_tokenized.append(gt_token_list)
+
+    eval_result = calculate_metrics(pdt_tokenized, gt_tokenized, is_verbose=False)
+    return  eval_result

From b6a5f817bc2b988b8adb4be3e16222bb67dab7c9 Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Tue, 4 Mar 2025 18:25:06 +0000
Subject: [PATCH 023/112] increased max new tokens for cc-ocr

---
 lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml | 2 +-
 lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-es.yaml | 2 +-
 lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-fr.yaml | 2 +-
 lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-it.yaml | 2 +-
 lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ko.yaml | 2 +-
 lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-pt.yaml | 2 +-
 lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ru.yaml | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml
index c9113bb2a..9064d4a29 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml
@@ -25,7 +25,7 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 16
+  max_new_tokens: 40
   temperature: 0
   do_sample: False
 
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-es.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-es.yaml
index 4b248cc86..b96264eba 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-es.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-es.yaml
@@ -25,7 +25,7 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 16
+  max_new_tokens: 40
   temperature: 0
   do_sample: False
 
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-fr.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-fr.yaml
index 1c4b60b5e..a7a75963d 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-fr.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-fr.yaml
@@ -25,7 +25,7 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 16
+  max_new_tokens: 40
   temperature: 0
   do_sample: False
 
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-it.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-it.yaml
index c3354d3aa..19faee861 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-it.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-it.yaml
@@ -25,7 +25,7 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 16
+  max_new_tokens: 40
   temperature: 0
   do_sample: False
 
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ko.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ko.yaml
index 2e43d68b1..439c4ee48 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ko.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ko.yaml
@@ -25,7 +25,7 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 16
+  max_new_tokens: 40
   temperature: 0
   do_sample: False
 
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-pt.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-pt.yaml
index 9fc4c3577..8119acdab 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-pt.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-pt.yaml
@@ -25,7 +25,7 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 16
+  max_new_tokens: 40
   temperature: 0
   do_sample: False
 
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ru.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ru.yaml
index 6435c9198..8554b2d8e 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ru.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ru.yaml
@@ -25,7 +25,7 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 16
+  max_new_tokens: 40
   temperature: 0
   do_sample: False
 

From e9acfbbff925ba6a8103d039666f19f59a8bbc98 Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Wed, 5 Mar 2025 21:30:09 +0000
Subject: [PATCH 024/112] fix pixstral run in signle gpu

---
 lmms_eval/models/pixtral.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/lmms_eval/models/pixtral.py b/lmms_eval/models/pixtral.py
index 7d0173d17..1ba4f2dea 100644
--- a/lmms_eval/models/pixtral.py
+++ b/lmms_eval/models/pixtral.py
@@ -51,6 +51,8 @@ def __init__(
         device_map: str = "",
         use_cache: bool = True,
         max_img_per_msg: int = 10,
+        tokenizer_mode: str = "mistral",
+        gpu_memory_utilization: float = 0.7,
         **kwargs,
     ) -> None:
         super().__init__()
@@ -70,7 +72,9 @@ def __init__(
         if isinstance(dtype, str) and dtype != "auto":
             dtype = getattr(torch, dtype)
 
-        self._model = LLM(model=pretrained, tokenizer_mode="mistral", limit_mm_per_prompt={"image": max_img_per_msg}, gpu_memory_utilization=0.7, max_model_len=16384)
+        self._tokenizer_mode = tokenizer_mode
+        self._gpu_memory_utilization = gpu_memory_utilization
+        self._model = LLM(model=pretrained, tokenizer_mode=self._tokenizer_mode, limit_mm_per_prompt={"image": max_img_per_msg}, gpu_memory_utilization=self._gpu_memory_utilization, max_model_len=16384)
         
         self.batch_size_per_gpu = int(batch_size)
         self.use_cache = use_cache
@@ -103,7 +107,7 @@ def __init__(
                 eval_logger.info("Using pipeline parallelism")
             else:
                 eval_logger.info(f"Using single device: {self._device}")
-                self.model.to(self._device)
+                # self.model.to(self._device)
             self._rank = 0
             self._world_size = 1
 

From 809c62c24a30ee2bfe4b1949d4ac8391d1145505 Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel.com>
Date: Thu, 6 Mar 2025 10:20:22 +0000
Subject: [PATCH 025/112] added the subset of the multilingual tasks that are
 relevant to tower

---
 lmms_eval/tasks/m3exam/m3exam_tower.yaml | 6 ++++++
 lmms_eval/tasks/maxm/maxm_tower.yaml     | 5 +++++
 lmms_eval/tasks/xgqa/xgqa_tower.yaml     | 8 ++++++++
 3 files changed, 19 insertions(+)
 create mode 100644 lmms_eval/tasks/m3exam/m3exam_tower.yaml
 create mode 100755 lmms_eval/tasks/maxm/maxm_tower.yaml
 create mode 100755 lmms_eval/tasks/xgqa/xgqa_tower.yaml

diff --git a/lmms_eval/tasks/m3exam/m3exam_tower.yaml b/lmms_eval/tasks/m3exam/m3exam_tower.yaml
new file mode 100644
index 000000000..cfcfb19fc
--- /dev/null
+++ b/lmms_eval/tasks/m3exam/m3exam_tower.yaml
@@ -0,0 +1,6 @@
+group: m3exam_tower
+task:
+- m3exam_chinese
+- m3exam_english
+- m3exam_italian
+- m3exam_portuguese
\ No newline at end of file
diff --git a/lmms_eval/tasks/maxm/maxm_tower.yaml b/lmms_eval/tasks/maxm/maxm_tower.yaml
new file mode 100755
index 000000000..220e9cd48
--- /dev/null
+++ b/lmms_eval/tasks/maxm/maxm_tower.yaml
@@ -0,0 +1,5 @@
+group : maxm
+task:
+  - maxm_en
+  - maxm_fr
+  - maxm_zh
\ No newline at end of file
diff --git a/lmms_eval/tasks/xgqa/xgqa_tower.yaml b/lmms_eval/tasks/xgqa/xgqa_tower.yaml
new file mode 100755
index 000000000..d5593256d
--- /dev/null
+++ b/lmms_eval/tasks/xgqa/xgqa_tower.yaml
@@ -0,0 +1,8 @@
+group: xgqa
+task:
+  - xgqa_de
+  - xgqa_en
+  - xgqa_ko
+  - xgqa_pt
+  - xgqa_ru
+  - xgqa_zh
\ No newline at end of file

From a310b1d0cb6af85a73498725b5876fb9f3aa18aa Mon Sep 17 00:00:00 2001
From: sonalsannigrahi <sannigrahisonal@gmail.com>
Date: Thu, 6 Mar 2025 10:44:34 +0000
Subject: [PATCH 026/112] Add ALM bench

---
 lmms_eval/tasks/alm_bench/alm_bench.yaml | 28 +++++++++++++
 lmms_eval/tasks/alm_bench/utils.py       | 53 ++++++++++++++++++++++++
 2 files changed, 81 insertions(+)
 create mode 100644 lmms_eval/tasks/alm_bench/alm_bench.yaml
 create mode 100644 lmms_eval/tasks/alm_bench/utils.py

diff --git a/lmms_eval/tasks/alm_bench/alm_bench.yaml b/lmms_eval/tasks/alm_bench/alm_bench.yaml
new file mode 100644
index 000000000..54d67a80e
--- /dev/null
+++ b/lmms_eval/tasks/alm_bench/alm_bench.yaml
@@ -0,0 +1,28 @@
+dataset_path: MBZUAI/ALM-Bench
+task: "alm-bench"
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.alm_bench_doc_to_visual
+doc_to_text: !function utils.alm_bench_doc_to_text
+doc_to_target: !function utils.alm_bench_doc_to_target
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  do_sample: False
+process_results: !function utils.alm_bench_process_results
+metric_list:
+  - metric: mcq
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  - version: 0.0
+model_specific_target_kwargs:
+  default: "mcq"
+lmms_eval_specific_kwargs:
+  default:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/utils.py b/lmms_eval/tasks/alm_bench/utils.py
new file mode 100644
index 000000000..665f345be
--- /dev/null
+++ b/lmms_eval/tasks/alm_bench/utils.py
@@ -0,0 +1,53 @@
+from PIL import Image
+
+
+def exact_match(pred, target):
+    if pred == target:
+        return 1
+    else:
+        return 0
+    
+def alm_bench_doc_to_visual(doc):
+    image = (doc['file_name']).convert('RGB')
+    return [image]
+
+
+def alm_bench_doc_to_text(doc, lmms_eval_specific_kwargs):
+    # Process MCQ question to extract answer, at the moment since we are not filtered the data set, I only process MCQ question types
+    question = doc["Translated_Question"]
+    pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+    post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+    pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    if lmms_eval_specific_kwargs["prompt_format"] == "mcq" and doc["Question_Type"] == "Multiple Choice Questions":
+        choices = doc["Translated_Answer"].split("(Options: ")[1].rstrip(")").split(", ")
+        choices_str = "\n".join([f"{choice}" for choice in choices])
+        print(f"{pre_prompt}{question}\n{choices_str}{post_prompt}")
+        return f"{pre_prompt}{question}\n{choices_str}{post_prompt}"
+    # elif lmms_eval_specific_kwargs["prompt_format"] == "tf" and doc["Question_Type"] == "True False Question":
+    #     options = "\n".join(["true","false"])
+    #     return f"{pre_prompt}{question}{options}{post_prompt}"
+    else:
+        return "WRONG PROMPT. Write no answer. "
+
+
+def alm_bench_process_results(doc, results):
+    pred = results[0]
+    type = doc["Question_Type"]
+    return_dict = {"mcq": 0, "others": 0}
+    if type == "Multiple Choice Questions":
+        target = doc["Translated_Answer"].split(" (Options: ")[0]
+        match = exact_match(pred, target)
+        return_dict["mcq"] = match
+    else:
+        return_dict["others"] = 0
+    return return_dict
+
+
+def alm_bench_doc_to_target(doc, model_specific_target_kwargs):
+    if model_specific_target_kwargs == "mcq":
+        return doc["Translated_Answer"].split(" (Options: ")[0]
+    # elif model_specific_target_kwargs == "tf":
+    #     return doc["Translated_Answer"]
+    else:
+        return "NO ANSWER"

From 33b56f3ceebe0425b6d0fc097dde341ac9e72a05 Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Fri, 7 Mar 2025 12:44:00 +0000
Subject: [PATCH 027/112] updated cc-ocr eval

---
 .../tasks/cc_ocr/cc-ocr-multi-lan-de.yaml      |  6 +++++-
 .../tasks/cc_ocr/cc-ocr-multi-lan-es.yaml      |  5 ++++-
 .../tasks/cc_ocr/cc-ocr-multi-lan-fr.yaml      |  5 ++++-
 .../tasks/cc_ocr/cc-ocr-multi-lan-it.yaml      |  5 ++++-
 .../tasks/cc_ocr/cc-ocr-multi-lan-ko.yaml      |  5 ++++-
 .../tasks/cc_ocr/cc-ocr-multi-lan-pt.yaml      |  5 ++++-
 .../tasks/cc_ocr/cc-ocr-multi-lan-ru.yaml      |  5 ++++-
 lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan.yaml   |  4 ++--
 lmms_eval/tasks/cc_ocr/cc_ocr_eval/utils.py    |  4 ++--
 lmms_eval/tasks/cc_ocr/utils.py                | 18 +++++++++++++-----
 10 files changed, 46 insertions(+), 16 deletions(-)

diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml
index 9064d4a29..9553eb180 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml
@@ -25,11 +25,15 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 40
+  max_new_tokens: 128
   temperature: 0
   do_sample: False
 
 
+lmms_eval_specific_kwargs:
+  default:
+    post_prompt: "\nAnswer by providing the text directly."
+
 metric_list:
   - metric: ocr_results
     aggregation: !function utils.ccocr_multi_lan_ocr_aggregate_results
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-es.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-es.yaml
index b96264eba..64c5db5de 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-es.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-es.yaml
@@ -25,10 +25,13 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 40
+  max_new_tokens: 128
   temperature: 0
   do_sample: False
 
+lmms_eval_specific_kwargs:
+  default:
+    post_prompt: "\nAnswer by providing the text directly."
 
 metric_list:
   - metric: ocr_results
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-fr.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-fr.yaml
index a7a75963d..0ebe05ee4 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-fr.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-fr.yaml
@@ -25,10 +25,13 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 40
+  max_new_tokens: 128
   temperature: 0
   do_sample: False
 
+lmms_eval_specific_kwargs:
+  default:
+    post_prompt: "\nAnswer by providing the text directly."
 
 metric_list:
   - metric: ocr_results
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-it.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-it.yaml
index 19faee861..9db9bba44 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-it.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-it.yaml
@@ -25,10 +25,13 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 40
+  max_new_tokens: 128
   temperature: 0
   do_sample: False
 
+lmms_eval_specific_kwargs:
+  default:
+    post_prompt: "\nAnswer by providing the text directly."
 
 metric_list:
   - metric: ocr_results
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ko.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ko.yaml
index 439c4ee48..a5fedf42e 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ko.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ko.yaml
@@ -25,10 +25,13 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 40
+  max_new_tokens: 128
   temperature: 0
   do_sample: False
 
+lmms_eval_specific_kwargs:
+  default:
+    post_prompt: "\nAnswer by providing the text directly."
 
 metric_list:
   - metric: ocr_results
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-pt.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-pt.yaml
index 8119acdab..91d897ab6 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-pt.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-pt.yaml
@@ -25,10 +25,13 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 40
+  max_new_tokens: 128
   temperature: 0
   do_sample: False
 
+lmms_eval_specific_kwargs:
+  default:
+    post_prompt: "\nAnswer by providing the text directly."
 
 metric_list:
   - metric: ocr_results
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ru.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ru.yaml
index 8554b2d8e..a39630f83 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ru.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ru.yaml
@@ -25,10 +25,13 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 40
+  max_new_tokens: 128
   temperature: 0
   do_sample: False
 
+lmms_eval_specific_kwargs:
+  default:
+    post_prompt: "\nAnswer by providing the text directly."
 
 metric_list:
   - metric: ocr_results
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan.yaml
index 1a63d5f56..6b4c0804e 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan.yaml
@@ -1,8 +1,8 @@
 group: cc-ocr-multi-lan
 task:
   - cc-ocr-multi-lan-de
-  - cc-ocr-multi-lan-fr
+  # - cc-ocr-multi-lan-fr
   # - cc-ocr-multi-lan-it
   # - cc-ocr-multi-lan-ru
   # - cc-ocr-multi-lan-es
-  - cc-ocr-multi-lan-ko
\ No newline at end of file
+  # - cc-ocr-multi-lan-ko
\ No newline at end of file
diff --git a/lmms_eval/tasks/cc_ocr/cc_ocr_eval/utils.py b/lmms_eval/tasks/cc_ocr/cc_ocr_eval/utils.py
index 88d0c149d..573a34a47 100644
--- a/lmms_eval/tasks/cc_ocr/cc_ocr_eval/utils.py
+++ b/lmms_eval/tasks/cc_ocr/cc_ocr_eval/utils.py
@@ -64,7 +64,7 @@ def calculate_metrics(predictions, groundtruths, is_verbose=False):
     hmean = 2 * recall_acc * preci_acc / (recall_acc + preci_acc + 1e-9)
     vbs_eval_result = {
         'macro_recall': final_macro_recall, 'macro_precision': final_macro_precision, 'macro_f1_score': final_macro_f1,
-        'micro_recall': recall_acc, 'micro_precision': preci_acc, 'mirco_f1_score': hmean
+        'micro_recall': recall_acc, 'micro_precision': preci_acc, 'micro_f1_score': hmean
     }
-    eval_result = vbs_eval_result if is_verbose else {'macro_f1_score': final_macro_f1, 'mirco_f1_score': hmean}
+    eval_result = vbs_eval_result if is_verbose else {'macro_f1_score': final_macro_f1, 'micro_f1_score': hmean}
     return eval_result
\ No newline at end of file
diff --git a/lmms_eval/tasks/cc_ocr/utils.py b/lmms_eval/tasks/cc_ocr/utils.py
index bf2302326..1e5c2851b 100644
--- a/lmms_eval/tasks/cc_ocr/utils.py
+++ b/lmms_eval/tasks/cc_ocr/utils.py
@@ -28,6 +28,7 @@ def ccocr_process_docs(docs):
     """
     logger.info(f"Converting base64 images to PIL Images...")
     # Process images in place
+    # docs = docs.select(range(10)) filter out some samples!
     docs = docs.map(
         lambda doc: {
             'image': base64_to_bytes(doc['image'])
@@ -41,9 +42,17 @@ def ccocr_doc_to_visual(doc):
     return [image]
 
 
-def ccocr_doc_to_text(doc, ):
+def ccocr_doc_to_text(doc,lmms_eval_specific_kwargs=None ):
     question = doc["question"]
-    return f"{question}"
+    if 'pre_prompt' in lmms_eval_specific_kwargs:
+        pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    else:
+        pre_prompt = ""
+    if 'post_prompt' in lmms_eval_specific_kwargs:
+        post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+    else:
+        post_prompt = ""
+    return f"{pre_prompt}{question}{post_prompt}"
 
 
 def ccocr_process_results(doc, results):
@@ -67,6 +76,7 @@ def ccocr_multi_lan_ocr_aggregate_results(results):
 
     non_basic_languages = ["Arabic", "Japanese", "Korean", "Chinese"]
     pdt_tokenized, gt_tokenized = [], []
+    
     for res  in results:
         index = res["index"]
         image_name = res["image_name"]
@@ -77,12 +87,10 @@ def ccocr_multi_lan_ocr_aggregate_results(results):
         language = image_name.split("_")[0]
         if language in non_basic_languages:
             is_word_level = False
-        
-        
         pdt_token_list = text_normalize_and_tokenize(pred.strip(), is_word_level, is_lower, is_alphanum_only)
         gt_token_list = text_normalize_and_tokenize(gt.strip(), is_word_level, is_lower, is_alphanum_only)
         pdt_tokenized.append(pdt_token_list)
         gt_tokenized.append(gt_token_list)
 
-    eval_result = calculate_metrics(pdt_tokenized, gt_tokenized, is_verbose=False)
+    eval_result = calculate_metrics(pdt_tokenized, gt_tokenized, is_verbose=True)
     return  eval_result

From 29765d066b7a257f5ec6c73b9a683533d31f6e21 Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel.com>
Date: Fri, 7 Mar 2025 15:12:51 +0000
Subject: [PATCH 028/112] fixed the print step

---
 scripts/parse_results.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/scripts/parse_results.py b/scripts/parse_results.py
index 70a73b116..53abdc9f2 100644
--- a/scripts/parse_results.py
+++ b/scripts/parse_results.py
@@ -63,7 +63,9 @@ def get_results_maxm(results, task_name, model_name):
 def get_results_mme(results, task_name, model_name):
     rows = []
     scores = {}
-    scores["en"] = f"{round(results['mme']['mme_cognition_score,none'],2)}/{round(results['mme']['mme_perception_score,none'],2)}"
+    mme_cognition_score = float(results['mme']['mme_cognition_score,none'])
+    mme_perception_score = float(results['mme']['mme_perception_score,none'])
+    scores["en"] = f"{round(mme_cognition_score,2)}/{round(mme_perception_score,2)}"
     
     for kk, vv in scores.items():
         rows.append({
@@ -248,8 +250,12 @@ def main():
     
     with open(input_file, "r") as file:
         data = json.load(file)
+        
     processed_data = process_results(data, task_name=args.task, model_name=model_name)
-    print(tabulate(processed_data, headers="keys"))
+    # print(tabulate(processed_data, headers="keys", tablefmt=","))
+    # print(processed_data)
+    for row in processed_data:
+        print(f"{row['model']},{row['task']},{row['language']},{row['score']}")
 
 if __name__ == "__main__":
     main()

From 6018cd26e75a23ef2498316831386f9c0007b3bf Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Fri, 7 Mar 2025 15:15:38 +0000
Subject: [PATCH 029/112] updated cc ocr task

---
 lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml |  2 +-
 lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan.yaml    | 11 ++++++-----
 lmms_eval/tasks/cc_ocr/utils.py                 |  2 +-
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml
index 9553eb180..538861744 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml
@@ -32,7 +32,7 @@ generation_kwargs:
 
 lmms_eval_specific_kwargs:
   default:
-    post_prompt: "\nAnswer by providing the text directly."
+    post_prompt: "\nAnswer by providing the text in the image directly."
 
 metric_list:
   - metric: ocr_results
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan.yaml
index 6b4c0804e..5d6abfb05 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan.yaml
@@ -1,8 +1,9 @@
 group: cc-ocr-multi-lan
 task:
   - cc-ocr-multi-lan-de
-  # - cc-ocr-multi-lan-fr
-  # - cc-ocr-multi-lan-it
-  # - cc-ocr-multi-lan-ru
-  # - cc-ocr-multi-lan-es
-  # - cc-ocr-multi-lan-ko
\ No newline at end of file
+  - cc-ocr-multi-lan-fr
+  - cc-ocr-multi-lan-it
+  - cc-ocr-multi-lan-ru
+  - cc-ocr-multi-lan-es
+  - cc-ocr-multi-lan-ko
+  - cc-ocr-multi-lan-pt
\ No newline at end of file
diff --git a/lmms_eval/tasks/cc_ocr/utils.py b/lmms_eval/tasks/cc_ocr/utils.py
index 1e5c2851b..917a8ce53 100644
--- a/lmms_eval/tasks/cc_ocr/utils.py
+++ b/lmms_eval/tasks/cc_ocr/utils.py
@@ -28,7 +28,7 @@ def ccocr_process_docs(docs):
     """
     logger.info(f"Converting base64 images to PIL Images...")
     # Process images in place
-    # docs = docs.select(range(10)) filter out some samples!
+    # docs = docs.select(range(20)) # filter out some samples!
     docs = docs.map(
         lambda doc: {
             'image': base64_to_bytes(doc['image'])

From 969410418237918d715aaaafe8870eb18846fdd6 Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Fri, 7 Mar 2025 15:16:46 +0000
Subject: [PATCH 030/112] minor changes to nvlm_d - warning when no image is
 included

---
 lmms_eval/models/nvlm_d.py | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/lmms_eval/models/nvlm_d.py b/lmms_eval/models/nvlm_d.py
index 72ebae9e1..7ede5b413 100644
--- a/lmms_eval/models/nvlm_d.py
+++ b/lmms_eval/models/nvlm_d.py
@@ -195,7 +195,7 @@ def _collate(x):
         for chunk in chunks:
             contexts, all_gen_kwargs, doc_to_visuals, doc_id, tasks, splits = zip(*chunk)
             visuals = [doc_to_visual(self.task_dict[task][split][ids]) for ids, task, split, doc_to_visual in zip(doc_id, tasks, splits, doc_to_visuals)]
-            
+
             # Use the generation kwargs from the first request in the batch
             gen_kwargs = all_gen_kwargs[0]
             
@@ -232,13 +232,17 @@ def _collate(x):
             assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
             context = contexts[0]
             visual = visuals[0]
-            # TODO: handle multiple images / understand the `visuals` object
             if isinstance(visual, list):
-                if len(visual) > 1:
-                    eval_logger.warning("More than one image is not supported for now... Using the first one")
-                visual = visual[0]
-            
-            if DEFAULT_IMAGE_TOKEN not in context:
+                if len(visual) == 0:
+                    eval_logger.warning("Found Sample with no image...")
+                    visual = None
+                elif len(visual) > 1:
+                    eval_logger.warning("More than one image is not supported for now... Using the first one for now.")
+                    visual = visual[0]
+                else:
+                    visual = visual[0]
+                
+            if DEFAULT_IMAGE_TOKEN not in context and visual is not None:
                 context = f"{DEFAULT_IMAGE_TOKEN}\n{context}"
             
             if self.tag:
@@ -249,7 +253,11 @@ def _collate(x):
                 chat.insert(0, {"role": "system", "content": self.add_system_prompt})
             prompt = self._tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)
             # Process inputs through the processor
-            inputs = self._processor(images=[visual], text=[prompt], return_tensors="pt")
+            if visual is None:
+                # TODO: handle this case where we have no image!
+                inputs = self._processor(images=[], text=[prompt], return_tensors="pt")
+            else:
+                inputs = self._processor(images=[visual], text=[prompt], return_tensors="pt")
             inputs = {k: v.to(self.device) for k, v in inputs.items()}
 
             # Generate outputs

From 3aeb15d541a82cf5ecd5f8beebf6b877a28677d0 Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Fri, 7 Mar 2025 19:07:47 +0000
Subject: [PATCH 031/112] updated cc-ocr eval

---
 lmms_eval/tasks/cc_ocr/utils.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/lmms_eval/tasks/cc_ocr/utils.py b/lmms_eval/tasks/cc_ocr/utils.py
index 917a8ce53..cfb50bc77 100644
--- a/lmms_eval/tasks/cc_ocr/utils.py
+++ b/lmms_eval/tasks/cc_ocr/utils.py
@@ -76,7 +76,7 @@ def ccocr_multi_lan_ocr_aggregate_results(results):
 
     non_basic_languages = ["Arabic", "Japanese", "Korean", "Chinese"]
     pdt_tokenized, gt_tokenized = [], []
-    
+
     for res  in results:
         index = res["index"]
         image_name = res["image_name"]
@@ -87,6 +87,11 @@ def ccocr_multi_lan_ocr_aggregate_results(results):
         language = image_name.split("_")[0]
         if language in non_basic_languages:
             is_word_level = False
+
+        if gt is None:
+            logger.warning(f"gt is None for index {index} and image_name {image_name}")
+            continue
+            
         pdt_token_list = text_normalize_and_tokenize(pred.strip(), is_word_level, is_lower, is_alphanum_only)
         gt_token_list = text_normalize_and_tokenize(gt.strip(), is_word_level, is_lower, is_alphanum_only)
         pdt_tokenized.append(pdt_token_list)

From e7ba1fa292d4ecb6c234787b194dbef91da08a55 Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Fri, 7 Mar 2025 19:46:20 +0000
Subject: [PATCH 032/112] init commit for commute--added

---
 lmms_eval/tasks/commute/comet_utils/comet.py  | 114 ++++
 .../commute/comet_utils/metrics_handler.py    |  75 +++
 .../tasks/commute/comet_utils/myutils.py      | 611 ++++++++++++++++++
 .../commute/comet_utils/result_handler.py     |  39 ++
 lmms_eval/tasks/commute/commute-all.yaml      |   8 +
 lmms_eval/tasks/commute/commute-ar.yaml       |  35 +
 lmms_eval/tasks/commute/commute-cs.yaml       |  35 +
 lmms_eval/tasks/commute/commute-de.yaml       |  35 +
 lmms_eval/tasks/commute/commute-es.yaml       |  35 +
 lmms_eval/tasks/commute/commute-fr.yaml       |  35 +
 lmms_eval/tasks/commute/commute-ru.yaml       |  35 +
 lmms_eval/tasks/commute/commute-zh.yaml       |  35 +
 lmms_eval/tasks/commute/utils.py              |  71 ++
 13 files changed, 1163 insertions(+)
 create mode 100644 lmms_eval/tasks/commute/comet_utils/comet.py
 create mode 100644 lmms_eval/tasks/commute/comet_utils/metrics_handler.py
 create mode 100644 lmms_eval/tasks/commute/comet_utils/myutils.py
 create mode 100644 lmms_eval/tasks/commute/comet_utils/result_handler.py
 create mode 100644 lmms_eval/tasks/commute/commute-all.yaml
 create mode 100644 lmms_eval/tasks/commute/commute-ar.yaml
 create mode 100644 lmms_eval/tasks/commute/commute-cs.yaml
 create mode 100644 lmms_eval/tasks/commute/commute-de.yaml
 create mode 100644 lmms_eval/tasks/commute/commute-es.yaml
 create mode 100644 lmms_eval/tasks/commute/commute-fr.yaml
 create mode 100644 lmms_eval/tasks/commute/commute-ru.yaml
 create mode 100644 lmms_eval/tasks/commute/commute-zh.yaml
 create mode 100644 lmms_eval/tasks/commute/utils.py

diff --git a/lmms_eval/tasks/commute/comet_utils/comet.py b/lmms_eval/tasks/commute/comet_utils/comet.py
new file mode 100644
index 000000000..e5cfe1faa
--- /dev/null
+++ b/lmms_eval/tasks/commute/comet_utils/comet.py
@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+from comet import download_model, load_from_checkpoint
+
+from lmms_eval.tasks.commute.comet_utils.metrics_handler import Metric
+from lmms_eval.tasks.commute.comet_utils.result_handler import MetricResult
+
+
+class BaseCOMETResult(MetricResult):
+    """
+    COMET Result Handler.
+    """
+
+    def __init__(
+        self,
+        result: float,
+    ) -> None:
+        super().__init__(result)
+
+
+class BaseCOMET(Metric):
+    def __init__(self, model: str, **kwargs) -> None:
+        super().__init__(**kwargs)
+        model_path = download_model(model)
+        self.model = load_from_checkpoint(model_path)
+        self.model.eval()
+
+    def load_gold_data(self, gold_data):
+        pass
+
+    def make_samples(
+        self, sources: list[str], hypotheses: list[str], references: list[str] = None
+    ):
+        pass
+
+    def run(
+        self,
+        hypothesis_path,
+        gold_data_path,
+        gpus: int = 1,
+        batch_size: int = 16,
+        **kwargs
+    ) -> dict:
+        hypotheses, gold_data = self._handle_inputs(hypothesis_path, gold_data_path)
+        references, sources = self.load_gold_data(gold_data)
+        result = self.evaluate(hypotheses, references, sources, gpus, batch_size)
+        result.print_result(self.metric_name())
+        return result.format_result(self.metric_name())
+
+    def evaluate(
+        self, hypotheses: list, references: list, sources: list, gpus, batch_size
+    ) -> BaseCOMETResult:
+        """
+        Evaluate function receives the hypotheses and the references and returns a COMETResult object.
+
+        :param hypotheses: List of the MT outputs (sentences).
+        :param references: List of the reference sentences.
+        :param sources: List of source sentences
+        """
+        samples = self.make_samples(sources, hypotheses, references)
+
+        outputs = self.model.predict(
+            samples=samples,
+            batch_size=batch_size,
+            gpus=gpus,
+            accelerator="auto",
+        )
+        system_score, segments_scores = outputs.system_score, outputs.scores
+
+        comet_result = BaseCOMETResult(
+            {
+                "system_score": system_score,
+                "segments_scores": segments_scores,
+            }
+        )
+        return comet_result
+
+    def process_result(self, result) -> MetricResult:
+        pass
+
+    @staticmethod
+    def metric_name():
+        pass
+
+
+class RefCOMET(BaseCOMET):
+    def __init__(self, model: str, **kwargs) -> None:
+        super().__init__(model, **kwargs)
+
+    def load_gold_data(self, gold_data):
+        references, sources = gold_data["ref"], gold_data["src"]
+        return references, sources
+
+    def make_samples(
+        self, sources: list[str], hypotheses: list[str], references: list[str]
+    ):
+        samples = {"src": sources, "mt": hypotheses, "ref": references}
+        samples = [dict(zip(samples, t)) for t in zip(*samples.values())]
+        return samples
+
+
+class QECOMET(BaseCOMET):
+    def __init__(self, model: str, **kwargs) -> None:
+        super().__init__(model, **kwargs)
+
+    def load_gold_data(self, gold_data):
+        references, sources = None, gold_data["src"]
+        return references, sources
+
+    def make_samples(
+        self, sources: list[str], hypotheses: list[str], references: list[str] = None
+    ):
+        samples = {"src": sources, "mt": hypotheses}
+        samples = [dict(zip(samples, t)) for t in zip(*samples.values())]
+        return samples
\ No newline at end of file
diff --git a/lmms_eval/tasks/commute/comet_utils/metrics_handler.py b/lmms_eval/tasks/commute/comet_utils/metrics_handler.py
new file mode 100644
index 000000000..29333f9d4
--- /dev/null
+++ b/lmms_eval/tasks/commute/comet_utils/metrics_handler.py
@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+from lmms_eval.tasks.commute.comet_utils.result_handler import MetricResult
+from lmms_eval.tasks.commute.comet_utils.myutils import list_to_dict, load_jsonl_file, read_lines
+
+
+class Metric(ABC):
+    """Abstract class defining a shared interface for all the Metrics"""
+
+    def __init__(self, **kwargs) -> None:
+        self.kwargs = kwargs
+
+    def run(self, hypothesis_path: Path = None, references_path: Path = None, **kwargs):
+        """
+        The runner function that performs all the necessary checks on the files,
+        applies the needed preprocessing (like lowercasing, tokenization, etc),
+        and calls the evaluate method of the class to get the scores.
+        """
+        pass
+
+    @abstractmethod
+    def evaluate(self, hypotheses: list, references: list, **kwargs) -> MetricResult:
+        """
+        Evaluate function receives the hypotheses and the reference files and returns a MetricResult object.
+
+        :param hypotheses: the path to the hypothese file.
+        :param references: the path to the reference file.
+        :return: MetricResult object.
+        """
+        pass
+
+    @abstractmethod
+    def process_result(self, result) -> MetricResult:
+        """
+        Process the result to be ready and complient with the MetricResult format.
+
+        :param result: the raw result produced by the metric
+        :return: MetricResult object.
+        """
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def metric_name() -> None:
+        """Metric name used to address the metric via cli."""
+        pass
+
+    @staticmethod
+    def _handle_inputs(
+        hypotheses: Path,
+        data_path: Path,
+    ) -> tuple:
+        """
+        Function to handle input files.
+        All inputs will be returned as list of strings.
+
+        :param hypotheses: either the handler to the file storing the hypotheses.
+        :param references: either the handler to the file storing the refereneces.
+        :param sources: either the handler to the file storing the source sentences.
+
+        :return:
+            -  Tuple with hypotheses, references and kwargs
+            -  If sources not None, Tuple with hypotheses, references, sources and kwargs
+        """
+        hypotheses = read_lines(hypotheses, unescape_newline=True)
+        # gold data keys depend on the task; e.g., for MT, it will include "ref", for APE "pe"
+        gold_data = load_jsonl_file(data_path)
+        assert len(hypotheses) == len(
+            gold_data
+        ), f"The number of hypotheses ({len(hypotheses)}) and rows in the gold data ({len(gold_data)}) should be the same."
+        gold_data = list_to_dict(gold_data)
+
+        return hypotheses, gold_data
\ No newline at end of file
diff --git a/lmms_eval/tasks/commute/comet_utils/myutils.py b/lmms_eval/tasks/commute/comet_utils/myutils.py
new file mode 100644
index 000000000..4adf7eaef
--- /dev/null
+++ b/lmms_eval/tasks/commute/comet_utils/myutils.py
@@ -0,0 +1,611 @@
+import argparse
+import ast
+import json
+import os
+import re
+import subprocess
+import time
+from pathlib import Path
+from typing import Dict, Iterable, List, Union
+
+import lxml
+import numpy as np
+import spacy
+import tqdm
+import yaml
+from loguru import logger
+from lxml import etree
+# from mosestokenizer import MosesDetokenizer
+from nltk.tokenize.treebank import TreebankWordDetokenizer
+from tenacity import Retrying, stop_after_attempt, wait_random_exponential
+
+PathInput = Union[str, Path]
+PATTERN_OPEN_TAG = r"<\s*([\w]+)\s*>\s*"
+PATTERN_CLOSE_TAG = r"\s*<\s*\/\s*([\w]+)\s*>"
+PATTERN_SHOT_NAME = r"^([0-9]+)[-_]shot.*$"
+
+SPACY_LANGUAGE_TO_MODEL = {
+    "en": "en_core_web_sm",
+    "de": "de_core_news_sm",
+    "fr": "fr_core_news_sm",
+    "nl": "nl_core_news_sm",
+    "pt": "pt_core_news_sm",
+    "ru": "ru_core_news_sm",
+    "it": "it_core_news_sm",
+    "pl": "pl_core_news_sm",
+    "es": "es_core_news_sm",
+    "sv": "sv_core_news_sm",
+    "zh": "zh_core_web_sm",
+    "ko": "ko_core_news_sm",
+}
+
+
+def args_to_dict(args, prefix: str, strip_prefix: bool = False):
+    """Filters argparse's `Namespace` into dictionary with arguments
+    beginning with the given prefix."""
+    prefix += "_"
+    d = {}
+    for k, v in args.__dict__.items():
+        if k.startswith(prefix) or k in ["language", "tokenize", "lowercase"]:
+            k = k.replace(prefix, "") if strip_prefix else k
+            d[k] = v
+    return d
+
+
+def detokenize(lines: list[str]) -> list[str]:
+    detokenizer = TreebankWordDetokenizer()
+    detokenized_lines = [detokenizer.detokenize(line.split()) for line in lines]
+    return detokenized_lines
+
+
+def detokenize_moses(lines: list[str], lang: str) -> list[str]:
+    detokenizer = MosesDetokenizer(lang)
+    detokenized_lines = [detokenizer(line.split()) for line in lines]
+    return detokenized_lines
+
+
+def tokenize_spacy(lines: list[str], language, keep_xml=False) -> list[str]:
+    spacy_model = SPACY_LANGUAGE_TO_MODEL[language]
+    try:
+        tokenizer = spacy.load(spacy_model, disable=["tagger", "parser", "ner"])
+    except OSError:
+        logger.warning(f"Spacy model {spacy_model} not found. Trying to download...")
+        download_proc = subprocess.run(
+            ["python", "-m", "spacy", "download", spacy_model]
+        )
+        tokenizer = spacy.load(spacy_model, disable=["tagger", "parser", "ner"])
+
+    tokenized = list(tokenizer.pipe(lines))
+    tokenized = [[token.text for token in sentence] for sentence in tokenized]
+    output_lines = [" ".join(sentence) for sentence in tokenized]
+    if keep_xml:
+        output_lines = [
+            re.sub(PATTERN_OPEN_TAG, r"<\1>", output_line)
+            for output_line in output_lines
+        ]
+        output_lines = [
+            re.sub(PATTERN_CLOSE_TAG, r"</\1>", output_line)
+            for output_line in output_lines
+        ]
+    return output_lines
+
+
+def parse_yaml_config(config_path):
+    with open(config_path, "r") as config_file:
+        config_data = yaml.safe_load(config_file)
+    return config_data
+
+
+def get_task_args(args: dict) -> dict:
+    source = args.get("source")
+    hypothesis = args.get("hypothesis")
+    references = args.get("references")
+
+    task_args = {
+        "hypothesis": hypothesis,
+        "references": references,
+    }
+    if source:
+        task_args.update({"source": source})
+    return task_args
+
+
+def load_json_file(
+    file_path: PathInput,
+) -> list:
+    """
+    Function to to load json files.
+
+    :param file_path: either the handler to the file storing the hypotheses.
+    """
+    with open(file_path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def load_jsonl_file(
+    file_path: PathInput,
+) -> list:
+    """
+    Function to parse error span jsonl files.
+    All inputs will be returned as list of dictionaries.
+
+    :param file_path: either the handler to the file storing the hypotheses.
+        Must be a jsonl file where each line is a dict which has at least the following keys:
+        {"mt": <mt>, "annotations": [{"start": <start>, "end": <end>, "severity": <severity>}, ...]}
+    :return:
+        -  List of dictionaries
+    """
+    with open(file_path, "r", encoding="utf-8") as f:
+        json_list = list(f)
+
+    return [json.loads(l) for l in json_list]
+
+
+def list_to_dict(l: list[dict[str, str]]) -> dict[str, list[str]]:
+    """
+    Assumes lists in dict have the same length.
+    """
+    d = {}
+    for k in l[0].keys():
+        d[k] = [d[k] for d in l]
+    return d
+
+
+def dict_to_records(d: dict[str, list[str]]) -> list[dict[str, str]]:
+    """
+    Assumes lists in dict have the same length.
+    """
+    records = []
+    for i in range(len(list(d.values())[0])):
+        records.append({k: v[i] for k, v in d.items()})
+
+    return records
+
+
+def write_lines(
+    path: PathInput,
+    lines: Iterable[str],
+    escape_newline: bool = False,
+    escape_return_char: bool = True,
+    verbose: bool = True,
+) -> None:
+    """Writes lines to a file.
+
+    Lines can be escaped, meaning \n is transformed to \\n.
+
+    Args:
+        path: The path to the file.
+        lines: The lines to write.
+        escape_newline: Whether to escape newlines.
+    """
+    # make dir, if not exists
+    if isinstance(path, str):
+        path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    out_lines = []
+    for i, line in enumerate(lines):
+        if escape_return_char:
+            if "\r" in line and verbose:
+                logger.opt(ansi=True).warning(
+                    f"Detected carriage return in line {i + 1} (\\r). This may cause errors downstream. Escaping. This behaviour is the default; you can turn it off with escape_return_char."
+                )
+            line = line.replace("\r", "\\r")
+        if escape_newline:
+            if "\n" in line and verbose:
+                logger.opt(ansi=True).warning(
+                    f"Found new line in line {i + 1} (\\n). This may cause errors downstream. Escaping."
+                )
+            line = line.replace("\n", "\\n")
+        out_lines.append(line)
+    with open(path, "w", encoding="utf-8") as f:
+        f.writelines((f"{l}\n" for l in out_lines))
+
+
+def combine_metrics_args(task_metric: dict, subtask_metric: dict) -> dict:
+    eval_metric = task_metric.copy()
+    for metric_arg, metric_arg_value in subtask_metric.items():
+        eval_metric[metric_arg] = metric_arg_value
+    return eval_metric
+
+
+def generate_with_retries(
+    retry_function,
+    model_args,
+    retry_max_attempts=1,
+    retry_multiplier=1,
+    retry_max_interval=10,
+    retry_min_interval=4,
+):
+    retryer = Retrying(
+        stop=stop_after_attempt(retry_max_attempts),
+        wait=wait_random_exponential(
+            multiplier=retry_multiplier, max=retry_max_interval, min=retry_min_interval
+        ),
+        reraise=True,
+    )
+    return retryer(retry_function, **model_args)
+
+
+def sample_strings_from_list(strings: list[str], num_samples: int) -> list[str]:
+    """ """
+    return np.random.choice(strings, num_samples).tolist()
+
+
+def read_lines(path: PathInput, unescape_newline: bool = False) -> List[str]:
+    """Reads lines from a file.
+    Lines can be unescapped, meaning \\n is transformed to \n.
+    Args:
+        path: The path to the file.
+        unescape_newline: Whether to unescape newlines.
+    Returns:
+        The lines in the file."""
+    with open(path, encoding="utf-8") as f:
+        lines = [l[:-1] for l in f.readlines()]
+    if unescape_newline:
+        lines = [l.replace("\\n", "\n") for l in lines]
+    return lines
+
+
+def make_dir_if_not_exists(output_file):
+    output_dir = os.path.dirname(output_file)
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+
+def log_response(response: str, step: int, lim: int = 10) -> None:
+    """
+    Logs the response from the model.
+    """
+    if step < lim:
+        if "<" in response:
+            response = response.replace("<", "\\<")
+            logger.opt(colors=True).info(
+                f"<red> Escaped tags in below response to not throw colorizing error. </red>"
+            )
+        logger.opt(colors=True).info(f"Response {step}: {response.strip()}")
+
+
+def get_num_processed_lines(output_file: str, **kwargs):
+    """
+    This function is used to write the output from the last line of the input file.
+    """
+    if os.path.exists(output_file):
+        processed_lines = read_lines(output_file, unescape_newline=False)
+        num_processed_lines = len(processed_lines)
+        if num_processed_lines > 0:
+            logger.opt(ansi=True).warning(
+                f"<red>Resuming generation from line {num_processed_lines}.</red>"
+            )
+    else:
+        num_processed_lines = 0
+
+    return num_processed_lines
+
+
+def load_data_to_records(
+    data_path: str,
+) -> list[dict[str, str]]:
+    """ """
+    # prompt data file can either be a jsonl or a csv
+    data = load_jsonl_file(data_path)
+
+    return data
+
+
+def save_to_json(
+    save_location: PathInput,
+    data: dict,
+) -> None:
+    """ """
+    if isinstance(save_location, str):
+        save_location = Path(save_location)
+    save_location.parent.mkdir(parents=True, exist_ok=True)
+    with open(save_location, "w") as output_path:
+        json.dump(data, output_path, indent=4)
+
+
+def get_sacrebleu_segment_scores(
+    hypotheses: list[str],
+    references,
+    method,
+) -> list[float]:
+    """ """
+    segment_scores = []
+    for h, r in zip(hypotheses, references):
+        segment_score = method.sentence_score(h, r)
+        segment_scores.append(segment_score.score)
+    return segment_scores
+
+
+def get_eval_args_given_task(
+    eval_args: Dict[str, str],
+    task_name: str,
+    subtask: str,
+    gen_output_dir: Path,
+    eval_data_dir: Path,
+    eval_output_dir: Path,
+    model_type: str,
+    model_name: str,
+) -> Dict[str, str]:
+    """ """
+    if task_name == "gec":
+        eval_args["references_m2"] = eval_data_dir / task_name / subtask / "test_corpus.m2"
+    gen_output_path = (
+        gen_output_dir
+        / task_name
+        / subtask
+        / model_type
+        / model_name
+        / "generation.txt"
+    )
+
+    eval_output_path = (
+        eval_output_dir
+        / task_name
+        / subtask
+        / model_type
+        / model_name
+        / "evaluation.json"
+    )
+
+    eval_data_path = eval_data_dir / task_name / subtask / "test.jsonl"
+    # add language argument to eval_args as it is needed in some metrics
+    lp = subtask.split(".")[1]
+    src_lang, trg_lang = get_langs(lp)
+    eval_args["lp"] = {"src_lang": src_lang, "trg_lang": trg_lang}
+
+    return gen_output_path, eval_data_path, eval_output_path, eval_args
+
+
+def text_to_label(
+    text: str,
+    text_type: str,
+    label_space: List[str] = None,
+    return_is_random: bool = False,
+) -> int:
+    is_random = False
+    if text_type == "categorical":
+        label = int(text.strip())
+        is_random = False
+    # if hypothesis is text
+    elif text_type == "text":
+        label = np.random.randint(len(label_space))
+        is_random = True
+        for l in label_space:
+            if re.search(l, text, re.IGNORECASE) is not None:
+                label = label_space.index(l)
+                is_random = False
+                break
+    if return_is_random:
+        return label, is_random
+    else:
+        return label
+
+
+def handle_subprocess(subprocess_args: List[str], check_output: bool = False):
+    output = 1
+    if check_output:
+        process = subprocess.Popen(
+            subprocess_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
+        )
+        output = ""
+
+        # Read and print the output in real-time
+        for line in iter(process.stdout.readline, ""):
+            print(line, end="")  # Print each line as it's received
+            output += line  # Append each line to the output variable
+    else:
+        try:
+            process = subprocess.Popen(subprocess_args)
+            while process.poll() is None:
+                time.sleep(1)
+            logger.info("Generation process has finished.")
+            output = 0
+        except KeyboardInterrupt:
+            process.terminate()
+    return output
+
+
+def tokenize_text(references, tokenizer):
+    tokenized_prompts = []
+    logger.info(f"Tokenizing text...")
+    for reference in tqdm.tqdm(references):
+        tokenized_prompts.append(tokenizer.encode(reference))
+    return tokenized_prompts
+
+
+def get_langs(lp):
+    # TODO: This has to be updated to account for more languages
+    valid_langs = [
+        "de",
+        "en-us",
+        "en-gb",
+        "en-uk",
+        "en",
+        "es-latam",
+        "es",
+        "fr",
+        "fr-ca",
+        "fr-fr",
+        "it",
+        "it-it",
+        "ko",
+        "ko-ko",
+        "nl",
+        "nl-nl",
+        "pl",
+        "pl-pl",
+        "pt-br",
+        "pt",
+        "pt-pt",
+        "ru",
+        "zh-CN",
+        "zh-cn",
+        "zh-TW",
+        "zh-tw",
+        "zh",
+        "ja",
+        "hi",
+        "uk",
+        "cs",
+        "is",
+        "sv",
+        "sv-se",
+        "ar",
+        "tr",
+        "el",
+        "hu",
+        "hu-hu",
+        "ro",
+        "ro-ro",
+        "fi",
+        "fi-fi",
+        "sk",
+        "da",
+        "da-da",
+        "lt",
+        "lv",
+        "sl",
+        "et",
+        "bg",
+        "no",
+        "no-no",
+        "ca",
+        "hr",
+        "ga",
+        "mt",
+        "gl",
+    ]
+    lang_pattern = "|".join(valid_langs)
+    lp_pattern = rf"^({lang_pattern})-({lang_pattern})$"
+    match = re.match(lp_pattern, lp)
+    if match:
+        # We have a language pair, so both src_lang and trg_lang will be extracted from lp
+        src_lang = match.group(1)
+        trg_lang = match.group(2)
+        return src_lang, trg_lang
+    elif lp in valid_langs:
+        # the task is monolingual, hence we only have one language. So, we set the src_lang only. trg_lang will be set to None
+        src_lang = lp
+        trg_lang = None
+        return src_lang, trg_lang
+    return None, None
+
+
+def add_average_generation_time(
+    input_file: str, metadata_file: str, language: str, mode: str = "lps"
+):
+    """
+    Given the generation file, the metadata file and the mode it calculates the time to generate the outputs.
+    Depending on the mode, it can be either the lines-per-second (lps) or words-per-second (wps)
+
+    Args:
+        input_file (str): The generation file
+        metadata_file (str): The metadata file that contains the generation_time which is a list
+        mode (str, optional): The mode for calculating the time. It can be either the lines-per-second (lps) or words-per-second (wps). Defaults to "lps".
+    """
+    lines = read_lines(input_file)
+    metadata = load_json_file(metadata_file)
+    generation_time = metadata.get("generation_time")
+    if generation_time:
+        total_time = sum(generation_time)
+        if mode == "lps":
+            average_time = len(lines) / total_time
+        # TODO: Add the support of wps mode
+    else:
+        logger.error(
+            f"The generation_time information doesn't exist in the {metadata_file}. We are going to set the average generation time to -1."
+        )
+        average_time = -1
+    metadata[f"generation_time_average ({mode})"] = average_time
+    save_to_json(metadata_file, metadata)
+
+
+def parse_dict_arg(arg):
+    try:
+        # Safely evaluate the string as a Python expression
+        parsed = ast.literal_eval(arg)
+        if isinstance(parsed, dict):
+            return parsed
+        else:
+            raise argparse.ArgumentTypeError("Argument is not a valid dictionary")
+    except (ValueError, SyntaxError):
+        raise argparse.ArgumentTypeError("Argument is not a valid dictionary")
+
+
+def write_jsonl(path, data):
+    with open(path, "w") as f:
+        for d in data:
+            f.write(json.dumps(d, ensure_ascii=False) + "\n")
+
+
+def match_xml(string_1: lxml.etree._Element, string_2: lxml.etree._Element):
+    if string_1.tag != string_2.tag:
+        return False
+
+    string_1 = list(string_1.iterchildren())
+    string_2 = list(string_2.iterchildren())
+
+    if len(string_1) != len(string_2):
+        return False
+
+    for s1, s2 in zip(string_1, string_2):
+        if not match_xml(s1, s2):
+            return False
+    return True
+
+
+def split_by_xml_tags(string: str):
+    try:
+        root = etree.fromstring(f"<root>{string}</root>")
+        parts = []
+
+        for element in root.iter():
+            if element.tag == "root":
+                if element.text:
+                    parts.append(element.text)
+                continue
+
+            if element.text:
+                parts.append(element.text)
+            if element.tail:
+                parts.append(element.tail)
+
+        return [p.strip() for p in parts if p.strip()]
+    except etree.XMLSyntaxError:
+        return [string]
+
+
+def split_pairs_by_xml_tags(string1: str, string2: str):
+    try:
+        string1_xml = etree.fromstring(f"<root>{string1}</root>")
+        string2_xml = etree.fromstring(f"<root>{string2}</root>")
+        if not match_xml(string1_xml, string2_xml):
+            return (None, None)
+
+        string1_segmented = split_by_xml_tags(string1)
+        string2_segmented = split_by_xml_tags(string2)
+        string1_segmented = [seg for seg in string1_segmented if seg is not None]
+        string2_segmented = [seg for seg in string2_segmented if seg is not None]
+
+        if len(string1_segmented) == len(string2_segmented):
+            return (string1_segmented, string2_segmented)
+    except:
+        pass
+
+    return (None, None)
+
+def prepare_xml_markup_pairs(hypotheses, references):
+    hypothesis_segmented, references_segmented = [], []
+    non_matching_indices = []
+    current_index = 0
+    for hyp, ref in zip(hypotheses, references):
+        hs, rs = split_pairs_by_xml_tags(hyp, ref)
+        if hs and rs:
+            hypothesis_segmented.extend(hs)
+            references_segmented.extend(rs)
+            current_index += len(hs)
+        else:
+            non_matching_indices.append(current_index)
+            current_index += 1
+    return hypothesis_segmented, references_segmented, non_matching_indices
\ No newline at end of file
diff --git a/lmms_eval/tasks/commute/comet_utils/result_handler.py b/lmms_eval/tasks/commute/comet_utils/result_handler.py
new file mode 100644
index 000000000..9b571d909
--- /dev/null
+++ b/lmms_eval/tasks/commute/comet_utils/result_handler.py
@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+from abc import ABC
+
+
+class MetricResult(ABC):
+    """
+    Abstract class defining a shared interface for all Metric Result Handlers.
+
+    :param result: float value to be displayed.
+    """
+
+    def __init__(self, result: float) -> None:
+        self.result = result
+
+    def print_result(self, metric_name: str, round_to_decimals: int = 4) -> None:
+        """Function used to display a particular Metric result.
+        :param metric_name: Metric name.
+        :param round_to_decimals: decimals that we want to present.
+        """
+        if type(self.result) == dict:
+            print(
+                f'{metric_name}: {round(self.result["system_score"], round_to_decimals)}'
+            )
+        else:
+            print(f"{metric_name}: {round(self.result, round_to_decimals)}")
+
+    def format_result(self, metric_name: str, round_to_decimals: int = 4) -> dict:
+        """Function used to format a particular Metric result.
+        :param metric_name: Metric name.
+        :param round_to_decimals: decimals that we want to present.
+        """
+        if type(self.result) == dict:
+            out = {
+                f"{metric_name}": round(self.result["system_score"], round_to_decimals),
+                f"{metric_name}_segments": self.result["segments_scores"],
+            }
+        else:
+            out = {f"{metric_name}": round(self.result, round_to_decimals)}
+        return out
\ No newline at end of file
diff --git a/lmms_eval/tasks/commute/commute-all.yaml b/lmms_eval/tasks/commute/commute-all.yaml
new file mode 100644
index 000000000..605f4be5e
--- /dev/null
+++ b/lmms_eval/tasks/commute/commute-all.yaml
@@ -0,0 +1,8 @@
+group: commute-all
+task:
+  - commute-de
+  - commute-ar
+  - commute-cs
+  - commute-fr
+  - commute-ru
+  - commute-zh
diff --git a/lmms_eval/tasks/commute/commute-ar.yaml b/lmms_eval/tasks/commute/commute-ar.yaml
new file mode 100644
index 000000000..9610f66b1
--- /dev/null
+++ b/lmms_eval/tasks/commute/commute-ar.yaml
@@ -0,0 +1,35 @@
+dataset_path:  Unbabel/commute_multimodal_mt
+task: commute-ar
+
+test_split: ar
+
+
+dataset_kwargs:
+  token: True
+
+
+process_docs:  !function utils.commute_process_docs
+doc_to_visual: !function utils.commute_doc_to_visual
+doc_to_text: !function utils.commute_doc_to_text
+process_results: !function utils.commute_process_results
+
+doc_to_target: "answer"
+
+output_type: generate_until
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  do_sample: False
+
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following text from English to Arabic by taking into account the image to resolve any ambiguities.\nEnglish: {source}\nArabic: "
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.commute_aggregate_results
+    higher_is_better: true
+
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/commute/commute-cs.yaml b/lmms_eval/tasks/commute/commute-cs.yaml
new file mode 100644
index 000000000..e59d444c2
--- /dev/null
+++ b/lmms_eval/tasks/commute/commute-cs.yaml
@@ -0,0 +1,35 @@
+dataset_path:  Unbabel/commute_multimodal_mt
+task: commute-cs
+
+test_split: cs
+
+
+dataset_kwargs:
+  token: True
+
+
+process_docs:  !function utils.commute_process_docs
+doc_to_visual: !function utils.commute_doc_to_visual
+doc_to_text: !function utils.commute_doc_to_text
+process_results: !function utils.commute_process_results
+
+doc_to_target: "answer"
+
+output_type: generate_until
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  do_sample: False
+
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following text from English to Czech by taking into account the image to resolve any ambiguities.\nEnglish: {source}\nCzech: "
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.commute_aggregate_results
+    higher_is_better: true
+
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/commute/commute-de.yaml b/lmms_eval/tasks/commute/commute-de.yaml
new file mode 100644
index 000000000..31cb645ab
--- /dev/null
+++ b/lmms_eval/tasks/commute/commute-de.yaml
@@ -0,0 +1,35 @@
+dataset_path:  Unbabel/commute_multimodal_mt
+task: commute-de
+
+test_split: de
+
+
+dataset_kwargs:
+  token: True
+
+
+process_docs:  !function utils.commute_process_docs
+doc_to_visual: !function utils.commute_doc_to_visual
+doc_to_text: !function utils.commute_doc_to_text
+process_results: !function utils.commute_process_results
+
+doc_to_target: "answer"
+
+output_type: generate_until
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  do_sample: False
+
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following text from English to German by taking into account the image to resolve any ambiguities.\nEnglish: {source}\nGerman: "
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.commute_aggregate_results
+    higher_is_better: true
+
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/commute/commute-es.yaml b/lmms_eval/tasks/commute/commute-es.yaml
new file mode 100644
index 000000000..81cee1d0a
--- /dev/null
+++ b/lmms_eval/tasks/commute/commute-es.yaml
@@ -0,0 +1,35 @@
+dataset_path:  Unbabel/commute_multimodal_mt
+task: commute-es
+
+test_split: es
+
+
+dataset_kwargs:
+  token: True
+
+
+process_docs:  !function utils.commute_process_docs
+doc_to_visual: !function utils.commute_doc_to_visual
+doc_to_text: !function utils.commute_doc_to_text
+process_results: !function utils.commute_process_results
+
+doc_to_target: "answer"
+
+output_type: generate_until
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  do_sample: False
+
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following text from English to Spanish by taking into account the image to resolve any ambiguities.\nEnglish: {source}\nSpanish: "
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.commute_aggregate_results
+    higher_is_better: true
+
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/commute/commute-fr.yaml b/lmms_eval/tasks/commute/commute-fr.yaml
new file mode 100644
index 000000000..db2e2d68a
--- /dev/null
+++ b/lmms_eval/tasks/commute/commute-fr.yaml
@@ -0,0 +1,35 @@
+dataset_path:  Unbabel/commute_multimodal_mt
+task: commute-fr
+
+test_split: fr
+
+
+dataset_kwargs:
+  token: True
+
+
+process_docs:  !function utils.commute_process_docs
+doc_to_visual: !function utils.commute_doc_to_visual
+doc_to_text: !function utils.commute_doc_to_text
+process_results: !function utils.commute_process_results
+
+doc_to_target: "answer"
+
+output_type: generate_until
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  do_sample: False
+
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following text from English to French by taking into account the image to resolve any ambiguities.\nEnglish: {source}\nFrench: "
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.commute_aggregate_results
+    higher_is_better: true
+
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/commute/commute-ru.yaml b/lmms_eval/tasks/commute/commute-ru.yaml
new file mode 100644
index 000000000..326962e8f
--- /dev/null
+++ b/lmms_eval/tasks/commute/commute-ru.yaml
@@ -0,0 +1,35 @@
+dataset_path:  Unbabel/commute_multimodal_mt
+task: commute-ru
+
+test_split: ru
+
+
+dataset_kwargs:
+  token: True
+
+
+process_docs:  !function utils.commute_process_docs
+doc_to_visual: !function utils.commute_doc_to_visual
+doc_to_text: !function utils.commute_doc_to_text
+process_results: !function utils.commute_process_results
+
+doc_to_target: "answer"
+
+output_type: generate_until
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  do_sample: False
+
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following text from English to Russian by taking into account the image to resolve any ambiguities.\nEnglish: {source}\nRussian: "
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.commute_aggregate_results
+    higher_is_better: true
+
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/commute/commute-zh.yaml b/lmms_eval/tasks/commute/commute-zh.yaml
new file mode 100644
index 000000000..f98a63918
--- /dev/null
+++ b/lmms_eval/tasks/commute/commute-zh.yaml
@@ -0,0 +1,35 @@
+dataset_path:  Unbabel/commute_multimodal_mt
+task: commute-zh
+
+test_split: zh
+
+
+dataset_kwargs:
+  token: True
+
+
+process_docs:  !function utils.commute_process_docs
+doc_to_visual: !function utils.commute_doc_to_visual
+doc_to_text: !function utils.commute_doc_to_text
+process_results: !function utils.commute_process_results
+
+doc_to_target: "answer"
+
+output_type: generate_until
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  do_sample: False
+
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following text from English to Chinese by taking into account the image to resolve any ambiguities.\nEnglish: {source}\nChinese: "
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.commute_aggregate_results
+    higher_is_better: true
+
+metadata:
+  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/commute/utils.py b/lmms_eval/tasks/commute/utils.py
new file mode 100644
index 000000000..a67f60cac
--- /dev/null
+++ b/lmms_eval/tasks/commute/utils.py
@@ -0,0 +1,71 @@
+
+import base64
+from PIL import Image
+from io import BytesIO
+import numpy as np
+
+from loguru import logger
+
+from lmms_eval.tasks.commute.comet_utils.comet import RefCOMET
+
+
+# def base64_to_bytes(base64_string):
+#     # Remove the header if it exists (e.g., "data:image/jpeg;base64,")
+#     if "base64," in base64_string:
+#         base64_string = base64_string.split("base64,")[1]
+#     # Decode base64 string to bytes
+#     img_bytes = base64.b64decode(base64_string)
+#     return img_bytes
+
+
+def commute_process_docs(docs):
+    # Process images in place
+    # docs = docs.select(range(20)) # filter out some samples!
+    return docs
+
+def commute_doc_to_visual(doc):
+    image = doc["image"].convert('RGB')
+    return [image]
+
+
+def commute_doc_to_text(doc,lmms_eval_specific_kwargs=None ):
+    source_txt = doc["source"]
+    pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    pre_prompt = pre_prompt.format(source=source_txt)
+    return f"{pre_prompt}"
+
+
+def commute_process_results(doc, results):
+    pred = results[0]
+    target = doc["correct_translation"]
+    incorrect_translation = doc["incorrect_translation"]
+    source = doc["source"]
+    return {"results": {"image_name": doc["image_name"], "source": source, "prediction": pred, "ground_truth": target, "incorrect_translation": incorrect_translation} }
+
+def compute_pairwise_accuracy(segments_scores_correct, segments_scores_incorrect):
+    pairwise_scores = []
+    for score_correct, score_incorrect in zip(segments_scores_correct, segments_scores_incorrect):
+        if score_correct > score_incorrect:
+            pairwise_scores.append(1)
+        else:
+            pairwise_scores.append(0)
+    pairwise_accuracy = sum(pairwise_scores) / len(pairwise_scores)
+    return pairwise_accuracy
+
+def commute_aggregate_results(results):
+    comet = RefCOMET(model="Unbabel/XCOMET-XL")
+    sources = [res["source"] for res in results]
+    hypotheses = [res["prediction"] for res in results]
+    references = [res["ground_truth"] for res in results]
+    incorrect_translations = [res["incorrect_translation"] for res in results]
+    comet.make_samples(sources, hypotheses, references)
+    segments_scores_correct = comet.evaluate(hypotheses, references, sources, gpus=1, batch_size=16).result["segments_scores"]
+    segments_scores_incorrect = comet.evaluate(hypotheses, incorrect_translations, sources, gpus=1, batch_size=16).result["segments_scores"]
+    pairwise_accuracy = compute_pairwise_accuracy(segments_scores_correct, segments_scores_incorrect)
+    results = {"pairwise_accuracy": pairwise_accuracy, 
+               "avg_correct_score": np.mean(segments_scores_correct), 
+               "avg_incorrect_score": np.mean(segments_scores_incorrect),
+            #    "segments_scores_correct": segments_scores_correct,
+            #    "segments_scores_incorrect": segments_scores_incorrect
+               }
+    return results

From c8158904201df3b3914803a7cc097d819c6ff20c Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Tue, 11 Mar 2025 18:38:19 +0000
Subject: [PATCH 033/112] updated parse results script for cc-ocr

---
 run_eval_aggregate.py | 53 +++++++++++++++++++++++++++++--------------
 1 file changed, 36 insertions(+), 17 deletions(-)

diff --git a/run_eval_aggregate.py b/run_eval_aggregate.py
index f2b20ae7e..12460358d 100644
--- a/run_eval_aggregate.py
+++ b/run_eval_aggregate.py
@@ -12,7 +12,8 @@
     "maxm":"maxm_en",
     "xm100":"xm100_en",
     "xmmmu":"mmmu_English_val",
-    "xgqa":"xgqa_en"
+    "xgqa":"xgqa_en",
+    'cc-ocr-multi-lan':None,
 }
 
 
@@ -39,22 +40,40 @@ def aggregate_results(data):
         main_task_dict_multi = {}
         main_task_dict_eng = {}
         for subtask in subtasks_dict[maintask_name]:
-            draft_dict = data["results"][subtask]
-            metric_names = [re.match(pattern, field).group() for field in draft_dict if re.match(pattern, field)]
-            if subtask==ENGLISH_TASKS[maintask_name]:
-                for metric in metric_names:
-                    value = draft_dict[f"{metric},none"]
-                    if metric not in main_task_dict_eng:
-                        main_task_dict_eng[metric] = [value]
-                    else:
-                        main_task_dict_eng[metric].append(value)
+            if "cc-ocr-multi-lan" in subtask:
+                draft_dict = data["results"][subtask]["ocr_results,none"]
+                metric_names = list(draft_dict.keys())
+                if subtask==ENGLISH_TASKS[maintask_name]:
+                    for metric in metric_names:
+                        value = draft_dict[f"{metric}"]
+                        if metric not in main_task_dict_eng:
+                            main_task_dict_eng[metric] = [value]
+                        else:
+                            main_task_dict_eng[metric].append(value)
+                else:
+                    for metric in metric_names:
+                        value = draft_dict[f"{metric}"]
+                        if metric not in main_task_dict_multi:
+                            main_task_dict_multi[metric] = [value]
+                        else:
+                            main_task_dict_multi[metric].append(value)
             else:
-                for metric in metric_names:
-                    value = draft_dict[f"{metric},none"]
-                    if metric not in main_task_dict_multi:
-                        main_task_dict_multi[metric] = [value]
-                    else:
-                        main_task_dict_multi[metric].append(value)
+                draft_dict = data["results"][subtask]
+                metric_names = [re.match(pattern, field).group() for field in draft_dict if re.match(pattern, field)]
+                if subtask==ENGLISH_TASKS[maintask_name]:
+                    for metric in metric_names:
+                        value = draft_dict[f"{metric},none"]
+                        if metric not in main_task_dict_eng:
+                            main_task_dict_eng[metric] = [value]
+                        else:
+                            main_task_dict_eng[metric].append(value)
+                else:
+                    for metric in metric_names:
+                        value = draft_dict[f"{metric},none"]
+                        if metric not in main_task_dict_multi:
+                            main_task_dict_multi[metric] = [value]
+                        else:
+                            main_task_dict_multi[metric].append(value)
         
         subtask_collect_multi_results[maintask_name] = main_task_dict_multi
         subtask_collect_eng_results[maintask_name] = main_task_dict_eng
@@ -64,7 +83,7 @@ def aggregate_results(data):
     for task_name in subtask_collect_multi_results:
         aggregated_results_multi[task_name] = {metric:np.mean(subtask_collect_multi_results[task_name][metric]) for metric in subtask_collect_multi_results[task_name] if metric!="submission"}
         aggregated_results_eng[task_name] = {metric:np.mean(subtask_collect_eng_results[task_name][metric]) for metric in subtask_collect_eng_results[task_name] if metric!="submission"}
-        
+
     return aggregated_results_eng, aggregated_results_multi
 
 

From 8842b0a1c5f7191de894725cdb7c48508a6a66e0 Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel.com>
Date: Wed, 12 Mar 2025 13:57:47 +0000
Subject: [PATCH 034/112] added the support for aya models

---
 lmms_eval/models/__init__.py |   1 +
 lmms_eval/models/aya.py      | 242 +++++++++++++++++++++++++++++++++++
 2 files changed, 243 insertions(+)
 create mode 100644 lmms_eval/models/aya.py

diff --git a/lmms_eval/models/__init__.py b/lmms_eval/models/__init__.py
index 6627a0cee..d848a5764 100644
--- a/lmms_eval/models/__init__.py
+++ b/lmms_eval/models/__init__.py
@@ -58,6 +58,7 @@
     "aria": "Aria",
     "pangea": "Pangea",
     "pixtral": "Pixtral",
+    "aya": "Aya",
 }
 
 
diff --git a/lmms_eval/models/aya.py b/lmms_eval/models/aya.py
new file mode 100644
index 000000000..09c17ed58
--- /dev/null
+++ b/lmms_eval/models/aya.py
@@ -0,0 +1,242 @@
+import warnings
+from typing import List, Optional, Tuple, Union
+import torch
+from accelerate import Accelerator, DistributedType
+from accelerate.state import AcceleratorState
+from tqdm import tqdm
+from lmms_eval.api.instance import Instance
+from lmms_eval.api.model import lmms
+from lmms_eval.api.registry import register_model
+
+import io
+import base64
+
+from transformers import AutoProcessor, AutoModelForImageTextToText
+
+warnings.filterwarnings("ignore")
+from loguru import logger as eval_logger
+
+DEFAULT_IMAGE_TOKEN = "<image>"
+
+@register_model("aya")
+class Aya(lmms):
+    """
+    Custom AYA model implementation using Hugging Face Transformers and remote code.
+
+    Example usage:
+
+    accelerate launch --num_processes=8 -m lmms_eval \
+        --model aya \
+        --model_args pretrained=CohereForAI/aya-vision-8b \
+        --tasks mme \
+        --batch_size 1 \
+        --output_path ./logs/ \
+        --log_samples
+    """
+
+    def __init__(
+        self,
+        pretrained: str = "CohereForAI/aya-vision-8b",
+        device: str = "cuda",
+        dtype: Optional[Union[str, torch.dtype]] = "bfloat16",
+        batch_size: int = 1,
+        device_map: str = "",
+        use_cache: bool = True,
+        **kwargs,
+    ) -> None:
+        super().__init__()
+        assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
+
+        accelerator = Accelerator()
+        if accelerator.num_processes > 1:
+            self._device = torch.device(f"cuda:{accelerator.local_process_index}")
+            self.device_map = f"cuda:{accelerator.local_process_index}"
+        elif accelerator.num_processes == 1 and device_map == "auto":
+            self._device = torch.device(device)
+            self.device_map = device_map
+        else:
+            self._device = torch.device(f"cuda:{accelerator.local_process_index}")
+            self.device_map = f"cuda:{accelerator.local_process_index}"
+
+        if isinstance(dtype, str) and dtype != "auto":
+            dtype = getattr(torch, dtype)
+
+        self._processor = AutoProcessor.from_pretrained(pretrained)
+        self._model = AutoModelForImageTextToText.from_pretrained(pretrained, device_map="auto", torch_dtype=torch.float16)
+        self.batch_size_per_gpu = int(batch_size)
+        self.use_cache = use_cache
+        # Handle distributed setup
+        if accelerator.num_processes > 1:
+            assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED]
+            
+            if accelerator.distributed_type == DistributedType.DEEPSPEED:
+                kwargs = {
+                    "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
+                    "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
+                }
+                AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
+                eval_logger.info("Using DeepSpeed - ensure zero stage is set to 0 in accelerate config")
+
+            if accelerator.distributed_type in [DistributedType.FSDP, DistributedType.DEEPSPEED]:
+                self._model = accelerator.prepare(self._model)
+            else:
+                self._model = accelerator.prepare_model(self._model, evaluation_mode=True)
+
+            self.accelerator = accelerator
+            if self.accelerator.is_local_main_process:
+                eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
+            self._rank = self.accelerator.local_process_index
+            self._world_size = self.accelerator.num_processes
+        else:
+            if device_map == "auto":
+                eval_logger.info("Using pipeline parallelism")
+            else:
+                eval_logger.info(f"Using single device: {self._device}")
+            self._rank = 0
+            self._world_size = 1
+
+    @property
+    def config(self):
+        return self._config
+
+    @property
+    def tokenizer(self):
+        return self._tokenizer
+
+    @property
+    def model(self):
+        if hasattr(self, "accelerator"):
+            return self.accelerator.unwrap_model(self._model)
+        else:
+            return self._model
+
+    @property
+    def eot_token_id(self):
+        return self.tokenizer.eos_token_id
+
+    @property
+    def max_length(self):
+        return self._max_length
+
+    @property
+    def batch_size(self):
+        return self.batch_size_per_gpu
+
+    @property
+    def device(self):
+        return self._device
+
+    @property
+    def rank(self):
+        return self._rank
+
+    @property
+    def world_size(self):
+        return self._world_size
+
+    def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
+        """Tokenize a string."""
+        add_special_tokens = False if add_special_tokens is None else add_special_tokens
+        encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
+        if left_truncate_len:
+            encoding = encoding[-left_truncate_len:]
+        return encoding
+
+    def tok_decode(self, tokens):
+        """Decode a sequence of tokens to a string."""
+        return self.tokenizer.decode(tokens)
+
+    def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
+        """Not implemented for NVLM model."""
+        raise NotImplementedError("Loglikelihood is not implemented for NVLM model")
+
+    def flatten(self, input):
+        """Flatten a nested list."""
+        new_list = []
+        for i in input:
+            for j in i:
+                new_list.append(j)
+        return new_list
+
+
+    def get_image_url(self, image_bytes: bytes) -> str:
+        buffer = io.BytesIO()
+        image_bytes.save(buffer, format="JPEG")
+        img_bytes = buffer.getvalue()
+        base64_image = base64.b64encode(img_bytes).decode('utf-8')
+        data_url = f"data:image/jpeg;base64,{base64_image}"
+        return data_url
+
+    def generate_until(self, requests: List[Instance]) -> List[str]:
+        """Generate text based on the given requests."""
+        res = []
+
+        def batch_requests(requests, batch_size):
+            for i in range(0, len(requests), batch_size):
+                yield [x.arguments for x in requests[i:i + batch_size]]
+
+        chunks = batch_requests(requests, self.batch_size)  
+        
+        num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
+        
+        pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
+        for chunk in chunks:
+            contexts, all_gen_kwargs, doc_to_visuals, doc_id, tasks, splits = zip(*chunk)
+            visuals = [doc_to_visual(self.task_dict[task][split][ids]) for ids, task, split, doc_to_visual in zip(doc_id, tasks, splits, doc_to_visuals)]
+            
+            # Use the generation kwargs from the first request in the batch
+            gen_kwargs = all_gen_kwargs[0]
+            
+            # Set default generation parameters if not provided
+            if "max_new_tokens" not in gen_kwargs:
+                gen_kwargs["max_new_tokens"] = 8192
+            if "temperature" not in gen_kwargs:
+                gen_kwargs["temperature"] = 0
+            if "do_sample" not in gen_kwargs:
+                gen_kwargs["do_sample"] = False
+
+            assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
+            context = contexts[0]
+            visual = visuals[0]
+            
+            # TODO: handle multiple images / understand the `visuals` object
+            if not isinstance(visual, list):
+                visual = [visual]
+
+            # vLLM does not work with bytes, so we need to convert it to a data url
+            image_urls = [self.get_image_url(v) for v in visual]
+            
+            # image_url = self.get_image_url(visual)
+
+            # Pixtral expects inputs in a different format, and doesn't work with <image> tokens added in the middle of the prompt.
+            context = context.replace(DEFAULT_IMAGE_TOKEN, "")
+
+            # create chat object
+            message = [
+                {"role": "user",
+                 "content": [
+                     {"type": "text", "text": context
+                      }]
+                 +
+                 [{"type": "image", "url": image_url} for image_url in image_urls]
+                     }]
+            
+            try:
+                inputs = self._processor.apply_chat_template(message, padding=True, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt").to(self._model.device)
+                result = self._model.generate(**inputs,
+                                             max_new_tokens=gen_kwargs["max_new_tokens"],
+                                             do_sample=gen_kwargs["do_sample"],
+                                             temperature=gen_kwargs["temperature"],
+                                             )
+                result = self._processor.tokenizer.decode(result[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
+            except Exception as e:
+                eval_logger.error(f"Error {e} in generating")
+                result = ""
+            res.append(result)
+            pbar.update(1)
+        pbar.close()
+        return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        """Not implemented for AYA model."""
+        raise NotImplementedError("Multi-round generation is not implemented for AYA model")
\ No newline at end of file

From 40f2c7e53e779450e741f28b451f201b9209ff20 Mon Sep 17 00:00:00 2001
From: farajian <amin@unbabel.com>
Date: Fri, 14 Mar 2025 19:20:32 +0000
Subject: [PATCH 035/112] changed the default conv_template to use qwen2
 instead

---
 lmms_eval/models/llava.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/lmms_eval/models/llava.py b/lmms_eval/models/llava.py
index 2de73cf5c..11ef8fcbf 100755
--- a/lmms_eval/models/llava.py
+++ b/lmms_eval/models/llava.py
@@ -60,7 +60,7 @@ def __init__(
         model_name=None,
         attn_implementation=best_fit_attn_implementation,
         device_map="cuda:0",
-        conv_template="vicuna_v1",
+        conv_template="qwen_2",
         use_cache=True,
         tie_weights: bool = True,
         truncate_context=False,  # whether to truncate the context in generation, set it False for LLaVA-1.6
@@ -70,7 +70,6 @@ def __init__(
         super().__init__()
         # Do not use kwargs for now
         assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
-
         accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
         accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
         self.accelerator = accelerator
@@ -213,7 +212,6 @@ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
         # TODO
         res = []
         pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
-
         for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
             # encode, pad, and truncate contexts for this batch
             if type(doc_to_target) == str:
@@ -286,7 +284,6 @@ def flatten(self, input):
 
     def generate_until(self, requests: List[Instance]) -> List[str]:
         res = []
-
         def _collate(x):
             # the negative sign on len(toks) sorts descending - this has a few advantages:
             # - time estimates will always be over not underestimates, which is more useful for planning
@@ -361,6 +358,10 @@ def _collate(x):
                     conv = copy.deepcopy(conv_templates[self.conv_template])
                 else:
                     conv = conv_templates[self.conv_template].copy()
+                # if conv.system:
+                #     conv.system = self.add_system_prompt
+                # elif self.add_system_prompt:
+                #     conv.system = self.add_system_prompt
                 conv.append_message(conv.roles[0], question)
                 conv.append_message(conv.roles[1], None)
                 prompt_question = conv.get_prompt()

From 441338e6c63a3737df4cb7acb4bfab0631cbbfd9 Mon Sep 17 00:00:00 2001
From: Sonal Sannigrahi <sannigrahisonal@gmail.com>
Date: Wed, 19 Mar 2025 17:11:00 +0000
Subject: [PATCH 036/112] add multi_30k

---
 lmms_eval/tasks/multi30k/multi30k.yaml | 36 ++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 lmms_eval/tasks/multi30k/multi30k.yaml

diff --git a/lmms_eval/tasks/multi30k/multi30k.yaml b/lmms_eval/tasks/multi30k/multi30k.yaml
new file mode 100644
index 000000000..c48437cd5
--- /dev/null
+++ b/lmms_eval/tasks/multi30k/multi30k.yaml
@@ -0,0 +1,36 @@
+dataset_path: romrawinjp/multi30k
+task: "multi30k"
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.multi30k_doc_to_visual
+doc_to_text: !function utils.multi30k_doc_to_text
+doc_to_target: "fr"
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.multi30k_process_results
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.multi30k_aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following caption from English to French by taking into account the image to resolve any ambiguities.\nEnglish: {source}\nFrench: "
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+

From 51dd52dd7c11e7236d22cd1126f56ef599b95d16 Mon Sep 17 00:00:00 2001
From: Sonal Sannigrahi <sannigrahisonal@gmail.com>
Date: Wed, 19 Mar 2025 17:11:32 +0000
Subject: [PATCH 037/112] add multi_30k

---
 lmms_eval/tasks/multi30k/utils.py | 33 +++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 lmms_eval/tasks/multi30k/utils.py

diff --git a/lmms_eval/tasks/multi30k/utils.py b/lmms_eval/tasks/multi30k/utils.py
new file mode 100644
index 000000000..f72368a64
--- /dev/null
+++ b/lmms_eval/tasks/multi30k/utils.py
@@ -0,0 +1,33 @@
+import numpy as np
+from loguru import logger as eval_logger
+from lmms_eval.tasks.commute.comet_utils.comet import RefCOMET
+
+
+
+def multi30k_doc_to_text(doc, lmms_eval_specific_kwargs):
+    source_txt = doc["en"]
+    pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    pre_prompt = pre_prompt.format(source=source_txt)
+    return f"{pre_prompt}"
+
+
+def multi30k_doc_to_visual(doc):
+    return [doc["image"].convert("RGB")]
+
+
+def multi30k_process_results(doc, results):
+    pred_answer = results[0]
+    target = doc["fr"]
+    return {"results":{"prediction":pred_answer,"ground_truth": target}}
+
+def multi30k_aggregate_results(results):
+    comet = RefCOMET(model="Unbabel/XCOMET-XL")
+    sources = [res["source"] for res in results]
+    hypotheses = [res["prediction"] for res in results]
+    references = [res["ground_truth"] for res in results]
+    comet.make_samples(sources, hypotheses, references)
+    segments_scores_correct = comet.evaluate(hypotheses, references, sources, gpus=1, batch_size=16).result["segments_scores"]
+    results = {
+               "avg_correct_score": np.mean(segments_scores_correct),
+               }
+    return results

From ed6c786606befd284b9cd63a7014c3430639b9c7 Mon Sep 17 00:00:00 2001
From: Sonal Sannigrahi <sannigrahisonal@gmail.com>
Date: Wed, 19 Mar 2025 17:13:55 +0000
Subject: [PATCH 038/112] fix source

---
 lmms_eval/tasks/multi30k/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lmms_eval/tasks/multi30k/utils.py b/lmms_eval/tasks/multi30k/utils.py
index f72368a64..818303c65 100644
--- a/lmms_eval/tasks/multi30k/utils.py
+++ b/lmms_eval/tasks/multi30k/utils.py
@@ -17,8 +17,9 @@ def multi30k_doc_to_visual(doc):
 
 def multi30k_process_results(doc, results):
     pred_answer = results[0]
+    source = doc["en"]
     target = doc["fr"]
-    return {"results":{"prediction":pred_answer,"ground_truth": target}}
+    return {"results":{"prediction":pred_answer,"ground_truth": target, "source":source}}
 
 def multi30k_aggregate_results(results):
     comet = RefCOMET(model="Unbabel/XCOMET-XL")

From eb9a51438bbe6973006b9274bc7f24ff9ec7d163 Mon Sep 17 00:00:00 2001
From: farajian <amin@unbabel.com>
Date: Wed, 19 Mar 2025 19:06:27 +0000
Subject: [PATCH 039/112] added system prompt to aya model

---
 lmms_eval/models/aya.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/lmms_eval/models/aya.py b/lmms_eval/models/aya.py
index 09c17ed58..feed9b589 100644
--- a/lmms_eval/models/aya.py
+++ b/lmms_eval/models/aya.py
@@ -40,6 +40,7 @@ def __init__(
         device: str = "cuda",
         dtype: Optional[Union[str, torch.dtype]] = "bfloat16",
         batch_size: int = 1,
+        add_system_prompt: Optional[str] = None,
         device_map: str = "",
         use_cache: bool = True,
         **kwargs,
@@ -65,6 +66,7 @@ def __init__(
         self._model = AutoModelForImageTextToText.from_pretrained(pretrained, device_map="auto", torch_dtype=torch.float16)
         self.batch_size_per_gpu = int(batch_size)
         self.use_cache = use_cache
+        self.add_system_prompt = add_system_prompt
         # Handle distributed setup
         if accelerator.num_processes > 1:
             assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED]
@@ -220,7 +222,11 @@ def batch_requests(requests, batch_size):
                  +
                  [{"type": "image", "url": image_url} for image_url in image_urls]
                      }]
-            
+
+            if self.add_system_prompt is not None:
+                message.insert(0, {"role": "system", 
+                                   "content": [{"type": "text", "text": self.add_system_prompt}]})
+                
             try:
                 inputs = self._processor.apply_chat_template(message, padding=True, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt").to(self._model.device)
                 result = self._model.generate(**inputs,

From d06352d6bba96870c4dd80c01cd10383f2b6f42c Mon Sep 17 00:00:00 2001
From: sonalsannigrahi <sannigrahisonal@gmail.com>
Date: Thu, 20 Mar 2025 05:11:27 +0000
Subject: [PATCH 040/112] update multi30_k

---
 lmms_eval/tasks/multi30k/comet_utils/comet.py |  98 +++
 .../multi30k/comet_utils/metrics_handler.py   |  75 +++
 .../tasks/multi30k/comet_utils/myutils.py     | 611 ++++++++++++++++++
 .../multi30k/comet_utils/result_handler.py    |  39 ++
 lmms_eval/tasks/multi30k/utils.py             |   8 +
 5 files changed, 831 insertions(+)
 create mode 100644 lmms_eval/tasks/multi30k/comet_utils/comet.py
 create mode 100644 lmms_eval/tasks/multi30k/comet_utils/metrics_handler.py
 create mode 100644 lmms_eval/tasks/multi30k/comet_utils/myutils.py
 create mode 100644 lmms_eval/tasks/multi30k/comet_utils/result_handler.py

diff --git a/lmms_eval/tasks/multi30k/comet_utils/comet.py b/lmms_eval/tasks/multi30k/comet_utils/comet.py
new file mode 100644
index 000000000..f88f4aee0
--- /dev/null
+++ b/lmms_eval/tasks/multi30k/comet_utils/comet.py
@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+from comet import download_model, load_from_checkpoint
+
+from lmms_eval.tasks.multi30k.comet_utils.metrics_handler import Metric
+from lmms_eval.tasks.multi30k.comet_utils.result_handler import MetricResult
+
+
+class BaseCOMETResult(MetricResult):
+    """
+    COMET Result Handler.
+    """
+
+    def __init__(
+        self,
+        result: float,
+    ) -> None:
+        super().__init__(result)
+
+
+class BaseCOMET(Metric):
+    def __init__(self, model: str, **kwargs) -> None:
+        super().__init__(**kwargs)
+        model_path = download_model(model)
+        self.model = load_from_checkpoint(model_path)
+        self.model.eval()
+
+    def load_gold_data(self, gold_data):
+        pass
+
+    def make_samples(
+        self, sources: list[str], hypotheses: list[str], references: list[str] = None
+    ):
+        pass
+
+    def run(
+        self,
+        hypothesis_path,
+        gold_data_path,
+        gpus: int = 1,
+        batch_size: int = 16,
+        **kwargs
+    ) -> dict:
+        hypotheses, gold_data = self._handle_inputs(hypothesis_path, gold_data_path)
+        references, sources = self.load_gold_data(gold_data)
+        result = self.evaluate(hypotheses, references, sources, gpus, batch_size)
+        result.print_result(self.metric_name())
+        return result.format_result(self.metric_name())
+
+    def evaluate(
+        self, hypotheses: list, references: list, sources: list, gpus, batch_size
+    ) -> BaseCOMETResult:
+        """
+        Evaluate function receives the hypotheses and the references and returns a COMETResult object.
+
+        :param hypotheses: List of the MT outputs (sentences).
+        :param references: List of the reference sentences.
+        :param sources: List of source sentences
+        """
+        samples = self.make_samples(sources, hypotheses, references)
+
+        outputs = self.model.predict(
+            samples=samples,
+            batch_size=batch_size,
+            gpus=gpus,
+            accelerator="auto",
+        )
+        system_score, segments_scores = outputs.system_score, outputs.scores
+
+        comet_result = BaseCOMETResult(
+            {
+                "system_score": system_score,
+                "segments_scores": segments_scores,
+            }
+        )
+        return comet_result
+
+    def process_result(self, result) -> MetricResult:
+        pass
+
+    @staticmethod
+    def metric_name():
+        pass
+
+
+class RefCOMET(BaseCOMET):
+    def __init__(self, model: str, **kwargs) -> None:
+        super().__init__(model, **kwargs)
+
+    def load_gold_data(self, gold_data):
+        references, sources = gold_data["ref"], gold_data["src"]
+        return references, sources
+
+    def make_samples(
+        self, sources: list[str], hypotheses: list[str], references: list[str]
+    ):
+        samples = {"src": sources, "mt": hypotheses, "ref": references}
+        samples = [dict(zip(samples, t)) for t in zip(*samples.values())]
+        return samples
\ No newline at end of file
diff --git a/lmms_eval/tasks/multi30k/comet_utils/metrics_handler.py b/lmms_eval/tasks/multi30k/comet_utils/metrics_handler.py
new file mode 100644
index 000000000..c8ab1560e
--- /dev/null
+++ b/lmms_eval/tasks/multi30k/comet_utils/metrics_handler.py
@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+from lmms_eval.tasks.multi30k.comet_utils.result_handler import MetricResult
+from lmms_eval.tasks.multi30k.comet_utils.myutils import list_to_dict, load_jsonl_file, read_lines
+
+
+class Metric(ABC):
+    """Abstract class defining a shared interface for all the Metrics"""
+
+    def __init__(self, **kwargs) -> None:
+        self.kwargs = kwargs
+
+    def run(self, hypothesis_path: Path = None, references_path: Path = None, **kwargs):
+        """
+        The runner function that performs all the necessary checks on the files,
+        applies the needed preprocessing (like lowercasing, tokenization, etc),
+        and calls the evaluate method of the class to get the scores.
+        """
+        pass
+
+    @abstractmethod
+    def evaluate(self, hypotheses: list, references: list, **kwargs) -> MetricResult:
+        """
+        Evaluate function receives the hypotheses and the reference files and returns a MetricResult object.
+
+        :param hypotheses: the path to the hypothese file.
+        :param references: the path to the reference file.
+        :return: MetricResult object.
+        """
+        pass
+
+    @abstractmethod
+    def process_result(self, result) -> MetricResult:
+        """
+        Process the result to be ready and complient with the MetricResult format.
+
+        :param result: the raw result produced by the metric
+        :return: MetricResult object.
+        """
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def metric_name() -> None:
+        """Metric name used to address the metric via cli."""
+        pass
+
+    @staticmethod
+    def _handle_inputs(
+        hypotheses: Path,
+        data_path: Path,
+    ) -> tuple:
+        """
+        Function to handle input files.
+        All inputs will be returned as list of strings.
+
+        :param hypotheses: either the handler to the file storing the hypotheses.
+        :param references: either the handler to the file storing the refereneces.
+        :param sources: either the handler to the file storing the source sentences.
+
+        :return:
+            -  Tuple with hypotheses, references and kwargs
+            -  If sources not None, Tuple with hypotheses, references, sources and kwargs
+        """
+        hypotheses = read_lines(hypotheses, unescape_newline=True)
+        # gold data keys depend on the task; e.g., for MT, it will include "ref", for APE "pe"
+        gold_data = load_jsonl_file(data_path)
+        assert len(hypotheses) == len(
+            gold_data
+        ), f"The number of hypotheses ({len(hypotheses)}) and rows in the gold data ({len(gold_data)}) should be the same."
+        gold_data = list_to_dict(gold_data)
+
+        return hypotheses, gold_data
\ No newline at end of file
diff --git a/lmms_eval/tasks/multi30k/comet_utils/myutils.py b/lmms_eval/tasks/multi30k/comet_utils/myutils.py
new file mode 100644
index 000000000..4adf7eaef
--- /dev/null
+++ b/lmms_eval/tasks/multi30k/comet_utils/myutils.py
@@ -0,0 +1,611 @@
+import argparse
+import ast
+import json
+import os
+import re
+import subprocess
+import time
+from pathlib import Path
+from typing import Dict, Iterable, List, Union
+
+import lxml
+import numpy as np
+import spacy
+import tqdm
+import yaml
+from loguru import logger
+from lxml import etree
+# from mosestokenizer import MosesDetokenizer
+from nltk.tokenize.treebank import TreebankWordDetokenizer
+from tenacity import Retrying, stop_after_attempt, wait_random_exponential
+
+PathInput = Union[str, Path]
+PATTERN_OPEN_TAG = r"<\s*([\w]+)\s*>\s*"
+PATTERN_CLOSE_TAG = r"\s*<\s*\/\s*([\w]+)\s*>"
+PATTERN_SHOT_NAME = r"^([0-9]+)[-_]shot.*$"
+
+SPACY_LANGUAGE_TO_MODEL = {
+    "en": "en_core_web_sm",
+    "de": "de_core_news_sm",
+    "fr": "fr_core_news_sm",
+    "nl": "nl_core_news_sm",
+    "pt": "pt_core_news_sm",
+    "ru": "ru_core_news_sm",
+    "it": "it_core_news_sm",
+    "pl": "pl_core_news_sm",
+    "es": "es_core_news_sm",
+    "sv": "sv_core_news_sm",
+    "zh": "zh_core_web_sm",
+    "ko": "ko_core_news_sm",
+}
+
+
+def args_to_dict(args, prefix: str, strip_prefix: bool = False):
+    """Filters argparse's `Namespace` into dictionary with arguments
+    beginning with the given prefix."""
+    prefix += "_"
+    d = {}
+    for k, v in args.__dict__.items():
+        if k.startswith(prefix) or k in ["language", "tokenize", "lowercase"]:
+            k = k.replace(prefix, "") if strip_prefix else k
+            d[k] = v
+    return d
+
+
+def detokenize(lines: list[str]) -> list[str]:
+    detokenizer = TreebankWordDetokenizer()
+    detokenized_lines = [detokenizer.detokenize(line.split()) for line in lines]
+    return detokenized_lines
+
+
+def detokenize_moses(lines: list[str], lang: str) -> list[str]:
+    detokenizer = MosesDetokenizer(lang)
+    detokenized_lines = [detokenizer(line.split()) for line in lines]
+    return detokenized_lines
+
+
+def tokenize_spacy(lines: list[str], language, keep_xml=False) -> list[str]:
+    spacy_model = SPACY_LANGUAGE_TO_MODEL[language]
+    try:
+        tokenizer = spacy.load(spacy_model, disable=["tagger", "parser", "ner"])
+    except OSError:
+        logger.warning(f"Spacy model {spacy_model} not found. Trying to download...")
+        download_proc = subprocess.run(
+            ["python", "-m", "spacy", "download", spacy_model]
+        )
+        tokenizer = spacy.load(spacy_model, disable=["tagger", "parser", "ner"])
+
+    tokenized = list(tokenizer.pipe(lines))
+    tokenized = [[token.text for token in sentence] for sentence in tokenized]
+    output_lines = [" ".join(sentence) for sentence in tokenized]
+    if keep_xml:
+        output_lines = [
+            re.sub(PATTERN_OPEN_TAG, r"<\1>", output_line)
+            for output_line in output_lines
+        ]
+        output_lines = [
+            re.sub(PATTERN_CLOSE_TAG, r"</\1>", output_line)
+            for output_line in output_lines
+        ]
+    return output_lines
+
+
+def parse_yaml_config(config_path):
+    with open(config_path, "r") as config_file:
+        config_data = yaml.safe_load(config_file)
+    return config_data
+
+
+def get_task_args(args: dict) -> dict:
+    source = args.get("source")
+    hypothesis = args.get("hypothesis")
+    references = args.get("references")
+
+    task_args = {
+        "hypothesis": hypothesis,
+        "references": references,
+    }
+    if source:
+        task_args.update({"source": source})
+    return task_args
+
+
+def load_json_file(
+    file_path: PathInput,
+) -> list:
+    """
+    Function to to load json files.
+
+    :param file_path: either the handler to the file storing the hypotheses.
+    """
+    with open(file_path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def load_jsonl_file(
+    file_path: PathInput,
+) -> list:
+    """
+    Function to parse error span jsonl files.
+    All inputs will be returned as list of dictionaries.
+
+    :param file_path: either the handler to the file storing the hypotheses.
+        Must be a jsonl file where each line is a dict which has at least the following keys:
+        {"mt": <mt>, "annotations": [{"start": <start>, "end": <end>, "severity": <severity>}, ...]}
+    :return:
+        -  List of dictionaries
+    """
+    with open(file_path, "r", encoding="utf-8") as f:
+        json_list = list(f)
+
+    return [json.loads(l) for l in json_list]
+
+
+def list_to_dict(l: list[dict[str, str]]) -> dict[str, list[str]]:
+    """
+    Assumes lists in dict have the same length.
+    """
+    d = {}
+    for k in l[0].keys():
+        d[k] = [d[k] for d in l]
+    return d
+
+
+def dict_to_records(d: dict[str, list[str]]) -> list[dict[str, str]]:
+    """
+    Assumes lists in dict have the same length.
+    """
+    records = []
+    for i in range(len(list(d.values())[0])):
+        records.append({k: v[i] for k, v in d.items()})
+
+    return records
+
+
+def write_lines(
+    path: PathInput,
+    lines: Iterable[str],
+    escape_newline: bool = False,
+    escape_return_char: bool = True,
+    verbose: bool = True,
+) -> None:
+    """Writes lines to a file.
+
+    Lines can be escaped, meaning \n is transformed to \\n.
+
+    Args:
+        path: The path to the file.
+        lines: The lines to write.
+        escape_newline: Whether to escape newlines.
+    """
+    # make dir, if not exists
+    if isinstance(path, str):
+        path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    out_lines = []
+    for i, line in enumerate(lines):
+        if escape_return_char:
+            if "\r" in line and verbose:
+                logger.opt(ansi=True).warning(
+                    f"Detected carriage return in line {i + 1} (\\r). This may cause errors downstream. Escaping. This behaviour is the default; you can turn it off with escape_return_char."
+                )
+            line = line.replace("\r", "\\r")
+        if escape_newline:
+            if "\n" in line and verbose:
+                logger.opt(ansi=True).warning(
+                    f"Found new line in line {i + 1} (\\n). This may cause errors downstream. Escaping."
+                )
+            line = line.replace("\n", "\\n")
+        out_lines.append(line)
+    with open(path, "w", encoding="utf-8") as f:
+        f.writelines((f"{l}\n" for l in out_lines))
+
+
+def combine_metrics_args(task_metric: dict, subtask_metric: dict) -> dict:
+    eval_metric = task_metric.copy()
+    for metric_arg, metric_arg_value in subtask_metric.items():
+        eval_metric[metric_arg] = metric_arg_value
+    return eval_metric
+
+
+def generate_with_retries(
+    retry_function,
+    model_args,
+    retry_max_attempts=1,
+    retry_multiplier=1,
+    retry_max_interval=10,
+    retry_min_interval=4,
+):
+    retryer = Retrying(
+        stop=stop_after_attempt(retry_max_attempts),
+        wait=wait_random_exponential(
+            multiplier=retry_multiplier, max=retry_max_interval, min=retry_min_interval
+        ),
+        reraise=True,
+    )
+    return retryer(retry_function, **model_args)
+
+
+def sample_strings_from_list(strings: list[str], num_samples: int) -> list[str]:
+    """ """
+    return np.random.choice(strings, num_samples).tolist()
+
+
+def read_lines(path: PathInput, unescape_newline: bool = False) -> List[str]:
+    """Reads lines from a file.
+    Lines can be unescapped, meaning \\n is transformed to \n.
+    Args:
+        path: The path to the file.
+        unescape_newline: Whether to unescape newlines.
+    Returns:
+        The lines in the file."""
+    with open(path, encoding="utf-8") as f:
+        lines = [l[:-1] for l in f.readlines()]
+    if unescape_newline:
+        lines = [l.replace("\\n", "\n") for l in lines]
+    return lines
+
+
+def make_dir_if_not_exists(output_file):
+    output_dir = os.path.dirname(output_file)
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+
+def log_response(response: str, step: int, lim: int = 10) -> None:
+    """
+    Logs the response from the model.
+    """
+    if step < lim:
+        if "<" in response:
+            response = response.replace("<", "\\<")
+            logger.opt(colors=True).info(
+                f"<red> Escaped tags in below response to not throw colorizing error. </red>"
+            )
+        logger.opt(colors=True).info(f"Response {step}: {response.strip()}")
+
+
+def get_num_processed_lines(output_file: str, **kwargs):
+    """
+    This function is used to write the output from the last line of the input file.
+    """
+    if os.path.exists(output_file):
+        processed_lines = read_lines(output_file, unescape_newline=False)
+        num_processed_lines = len(processed_lines)
+        if num_processed_lines > 0:
+            logger.opt(ansi=True).warning(
+                f"<red>Resuming generation from line {num_processed_lines}.</red>"
+            )
+    else:
+        num_processed_lines = 0
+
+    return num_processed_lines
+
+
+def load_data_to_records(
+    data_path: str,
+) -> list[dict[str, str]]:
+    """ """
+    # prompt data file can either be a jsonl or a csv
+    data = load_jsonl_file(data_path)
+
+    return data
+
+
+def save_to_json(
+    save_location: PathInput,
+    data: dict,
+) -> None:
+    """ """
+    if isinstance(save_location, str):
+        save_location = Path(save_location)
+    save_location.parent.mkdir(parents=True, exist_ok=True)
+    with open(save_location, "w") as output_path:
+        json.dump(data, output_path, indent=4)
+
+
+def get_sacrebleu_segment_scores(
+    hypotheses: list[str],
+    references,
+    method,
+) -> list[float]:
+    """ """
+    segment_scores = []
+    for h, r in zip(hypotheses, references):
+        segment_score = method.sentence_score(h, r)
+        segment_scores.append(segment_score.score)
+    return segment_scores
+
+
+def get_eval_args_given_task(
+    eval_args: Dict[str, str],
+    task_name: str,
+    subtask: str,
+    gen_output_dir: Path,
+    eval_data_dir: Path,
+    eval_output_dir: Path,
+    model_type: str,
+    model_name: str,
+) -> Dict[str, str]:
+    """ """
+    if task_name == "gec":
+        eval_args["references_m2"] = eval_data_dir / task_name / subtask / "test_corpus.m2"
+    gen_output_path = (
+        gen_output_dir
+        / task_name
+        / subtask
+        / model_type
+        / model_name
+        / "generation.txt"
+    )
+
+    eval_output_path = (
+        eval_output_dir
+        / task_name
+        / subtask
+        / model_type
+        / model_name
+        / "evaluation.json"
+    )
+
+    eval_data_path = eval_data_dir / task_name / subtask / "test.jsonl"
+    # add language argument to eval_args as it is needed in some metrics
+    lp = subtask.split(".")[1]
+    src_lang, trg_lang = get_langs(lp)
+    eval_args["lp"] = {"src_lang": src_lang, "trg_lang": trg_lang}
+
+    return gen_output_path, eval_data_path, eval_output_path, eval_args
+
+
+def text_to_label(
+    text: str,
+    text_type: str,
+    label_space: List[str] = None,
+    return_is_random: bool = False,
+) -> int:
+    is_random = False
+    if text_type == "categorical":
+        label = int(text.strip())
+        is_random = False
+    # if hypothesis is text
+    elif text_type == "text":
+        label = np.random.randint(len(label_space))
+        is_random = True
+        for l in label_space:
+            if re.search(l, text, re.IGNORECASE) is not None:
+                label = label_space.index(l)
+                is_random = False
+                break
+    if return_is_random:
+        return label, is_random
+    else:
+        return label
+
+
+def handle_subprocess(subprocess_args: List[str], check_output: bool = False):
+    output = 1
+    if check_output:
+        process = subprocess.Popen(
+            subprocess_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
+        )
+        output = ""
+
+        # Read and print the output in real-time
+        for line in iter(process.stdout.readline, ""):
+            print(line, end="")  # Print each line as it's received
+            output += line  # Append each line to the output variable
+    else:
+        try:
+            process = subprocess.Popen(subprocess_args)
+            while process.poll() is None:
+                time.sleep(1)
+            logger.info("Generation process has finished.")
+            output = 0
+        except KeyboardInterrupt:
+            process.terminate()
+    return output
+
+
+def tokenize_text(references, tokenizer):
+    tokenized_prompts = []
+    logger.info(f"Tokenizing text...")
+    for reference in tqdm.tqdm(references):
+        tokenized_prompts.append(tokenizer.encode(reference))
+    return tokenized_prompts
+
+
+def get_langs(lp):
+    # TODO: This has to be updated to account for more languages
+    valid_langs = [
+        "de",
+        "en-us",
+        "en-gb",
+        "en-uk",
+        "en",
+        "es-latam",
+        "es",
+        "fr",
+        "fr-ca",
+        "fr-fr",
+        "it",
+        "it-it",
+        "ko",
+        "ko-ko",
+        "nl",
+        "nl-nl",
+        "pl",
+        "pl-pl",
+        "pt-br",
+        "pt",
+        "pt-pt",
+        "ru",
+        "zh-CN",
+        "zh-cn",
+        "zh-TW",
+        "zh-tw",
+        "zh",
+        "ja",
+        "hi",
+        "uk",
+        "cs",
+        "is",
+        "sv",
+        "sv-se",
+        "ar",
+        "tr",
+        "el",
+        "hu",
+        "hu-hu",
+        "ro",
+        "ro-ro",
+        "fi",
+        "fi-fi",
+        "sk",
+        "da",
+        "da-da",
+        "lt",
+        "lv",
+        "sl",
+        "et",
+        "bg",
+        "no",
+        "no-no",
+        "ca",
+        "hr",
+        "ga",
+        "mt",
+        "gl",
+    ]
+    lang_pattern = "|".join(valid_langs)
+    lp_pattern = rf"^({lang_pattern})-({lang_pattern})$"
+    match = re.match(lp_pattern, lp)
+    if match:
+        # We have a language pair, so both src_lang and trg_lang will be extracted from lp
+        src_lang = match.group(1)
+        trg_lang = match.group(2)
+        return src_lang, trg_lang
+    elif lp in valid_langs:
+        # the task is monolingual, hence we only have one language. So, we set the src_lang only. trg_lang will be set to None
+        src_lang = lp
+        trg_lang = None
+        return src_lang, trg_lang
+    return None, None
+
+
+def add_average_generation_time(
+    input_file: str, metadata_file: str, language: str, mode: str = "lps"
+):
+    """
+    Given the generation file, the metadata file and the mode it calculates the time to generate the outputs.
+    Depending on the mode, it can be either the lines-per-second (lps) or words-per-second (wps)
+
+    Args:
+        input_file (str): The generation file
+        metadata_file (str): The metadata file that contains the generation_time which is a list
+        mode (str, optional): The mode for calculating the time. It can be either the lines-per-second (lps) or words-per-second (wps). Defaults to "lps".
+    """
+    lines = read_lines(input_file)
+    metadata = load_json_file(metadata_file)
+    generation_time = metadata.get("generation_time")
+    if generation_time:
+        total_time = sum(generation_time)
+        if mode == "lps":
+            average_time = len(lines) / total_time
+        # TODO: Add the support of wps mode
+    else:
+        logger.error(
+            f"The generation_time information doesn't exist in the {metadata_file}. We are going to set the average generation time to -1."
+        )
+        average_time = -1
+    metadata[f"generation_time_average ({mode})"] = average_time
+    save_to_json(metadata_file, metadata)
+
+
+def parse_dict_arg(arg):
+    try:
+        # Safely evaluate the string as a Python expression
+        parsed = ast.literal_eval(arg)
+        if isinstance(parsed, dict):
+            return parsed
+        else:
+            raise argparse.ArgumentTypeError("Argument is not a valid dictionary")
+    except (ValueError, SyntaxError):
+        raise argparse.ArgumentTypeError("Argument is not a valid dictionary")
+
+
+def write_jsonl(path, data):
+    with open(path, "w") as f:
+        for d in data:
+            f.write(json.dumps(d, ensure_ascii=False) + "\n")
+
+
+def match_xml(string_1: lxml.etree._Element, string_2: lxml.etree._Element):
+    if string_1.tag != string_2.tag:
+        return False
+
+    string_1 = list(string_1.iterchildren())
+    string_2 = list(string_2.iterchildren())
+
+    if len(string_1) != len(string_2):
+        return False
+
+    for s1, s2 in zip(string_1, string_2):
+        if not match_xml(s1, s2):
+            return False
+    return True
+
+
+def split_by_xml_tags(string: str):
+    try:
+        root = etree.fromstring(f"<root>{string}</root>")
+        parts = []
+
+        for element in root.iter():
+            if element.tag == "root":
+                if element.text:
+                    parts.append(element.text)
+                continue
+
+            if element.text:
+                parts.append(element.text)
+            if element.tail:
+                parts.append(element.tail)
+
+        return [p.strip() for p in parts if p.strip()]
+    except etree.XMLSyntaxError:
+        return [string]
+
+
+def split_pairs_by_xml_tags(string1: str, string2: str):
+    try:
+        string1_xml = etree.fromstring(f"<root>{string1}</root>")
+        string2_xml = etree.fromstring(f"<root>{string2}</root>")
+        if not match_xml(string1_xml, string2_xml):
+            return (None, None)
+
+        string1_segmented = split_by_xml_tags(string1)
+        string2_segmented = split_by_xml_tags(string2)
+        string1_segmented = [seg for seg in string1_segmented if seg is not None]
+        string2_segmented = [seg for seg in string2_segmented if seg is not None]
+
+        if len(string1_segmented) == len(string2_segmented):
+            return (string1_segmented, string2_segmented)
+    except:
+        pass
+
+    return (None, None)
+
+def prepare_xml_markup_pairs(hypotheses, references):
+    hypothesis_segmented, references_segmented = [], []
+    non_matching_indices = []
+    current_index = 0
+    for hyp, ref in zip(hypotheses, references):
+        hs, rs = split_pairs_by_xml_tags(hyp, ref)
+        if hs and rs:
+            hypothesis_segmented.extend(hs)
+            references_segmented.extend(rs)
+            current_index += len(hs)
+        else:
+            non_matching_indices.append(current_index)
+            current_index += 1
+    return hypothesis_segmented, references_segmented, non_matching_indices
\ No newline at end of file
diff --git a/lmms_eval/tasks/multi30k/comet_utils/result_handler.py b/lmms_eval/tasks/multi30k/comet_utils/result_handler.py
new file mode 100644
index 000000000..9b571d909
--- /dev/null
+++ b/lmms_eval/tasks/multi30k/comet_utils/result_handler.py
@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+from abc import ABC
+
+
+class MetricResult(ABC):
+    """
+    Abstract class defining a shared interface for all Metric Result Handlers.
+
+    :param result: float value to be displayed.
+    """
+
+    def __init__(self, result: float) -> None:
+        self.result = result
+
+    def print_result(self, metric_name: str, round_to_decimals: int = 4) -> None:
+        """Function used to display a particular Metric result.
+        :param metric_name: Metric name.
+        :param round_to_decimals: decimals that we want to present.
+        """
+        if type(self.result) == dict:
+            print(
+                f'{metric_name}: {round(self.result["system_score"], round_to_decimals)}'
+            )
+        else:
+            print(f"{metric_name}: {round(self.result, round_to_decimals)}")
+
+    def format_result(self, metric_name: str, round_to_decimals: int = 4) -> dict:
+        """Function used to format a particular Metric result.
+        :param metric_name: Metric name.
+        :param round_to_decimals: decimals that we want to present.
+        """
+        if type(self.result) == dict:
+            out = {
+                f"{metric_name}": round(self.result["system_score"], round_to_decimals),
+                f"{metric_name}_segments": self.result["segments_scores"],
+            }
+        else:
+            out = {f"{metric_name}": round(self.result, round_to_decimals)}
+        return out
\ No newline at end of file
diff --git a/lmms_eval/tasks/multi30k/utils.py b/lmms_eval/tasks/multi30k/utils.py
index 818303c65..a214f632e 100644
--- a/lmms_eval/tasks/multi30k/utils.py
+++ b/lmms_eval/tasks/multi30k/utils.py
@@ -1,6 +1,10 @@
 import numpy as np
 from loguru import logger as eval_logger
+<<<<<<< HEAD
 from lmms_eval.tasks.commute.comet_utils.comet import RefCOMET
+=======
+from lmms_eval.tasks.multi30k.comet_utils.comet import RefCOMET
+>>>>>>> f0da86b (update multi30_k)
 
 
 
@@ -31,4 +35,8 @@ def multi30k_aggregate_results(results):
     results = {
                "avg_correct_score": np.mean(segments_scores_correct),
                }
+<<<<<<< HEAD
     return results
+=======
+    return results
+>>>>>>> f0da86b (update multi30_k)

From a7cb1ad5523ad075214848f084bfb0624b004cf5 Mon Sep 17 00:00:00 2001
From: Sonal Sannigrahi <sannigrahisonal@gmail.com>
Date: Thu, 20 Mar 2025 05:21:00 +0000
Subject: [PATCH 041/112] fix merge conflicts

---
 lmms_eval/tasks/multi30k/utils.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/lmms_eval/tasks/multi30k/utils.py b/lmms_eval/tasks/multi30k/utils.py
index a214f632e..3ccdb6300 100644
--- a/lmms_eval/tasks/multi30k/utils.py
+++ b/lmms_eval/tasks/multi30k/utils.py
@@ -1,10 +1,6 @@
 import numpy as np
 from loguru import logger as eval_logger
-<<<<<<< HEAD
-from lmms_eval.tasks.commute.comet_utils.comet import RefCOMET
-=======
 from lmms_eval.tasks.multi30k.comet_utils.comet import RefCOMET
->>>>>>> f0da86b (update multi30_k)
 
 
 
@@ -35,8 +31,5 @@ def multi30k_aggregate_results(results):
     results = {
                "avg_correct_score": np.mean(segments_scores_correct),
                }
-<<<<<<< HEAD
     return results
-=======
-    return results
->>>>>>> f0da86b (update multi30_k)
+

From 3be37df318f115baddf78fa3d694b45a64230e3b Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Thu, 20 Mar 2025 12:34:33 +0000
Subject: [PATCH 042/112] updated commute task

---
 ...-all.yaml => commute-all-contrastive.yaml} |  2 +-
 lmms_eval/tasks/commute/commute-ar.yaml       | 12 +--
 lmms_eval/tasks/commute/commute-cs.yaml       | 12 +--
 lmms_eval/tasks/commute/commute-de.yaml       | 12 +--
 lmms_eval/tasks/commute/commute-es.yaml       | 12 +--
 lmms_eval/tasks/commute/commute-fr.yaml       | 12 +--
 lmms_eval/tasks/commute/commute-ru.yaml       | 12 +--
 lmms_eval/tasks/commute/commute-zh.yaml       | 12 +--
 lmms_eval/tasks/commute/utils.py              | 77 +++++++++++--------
 9 files changed, 89 insertions(+), 74 deletions(-)
 rename lmms_eval/tasks/commute/{commute-all.yaml => commute-all-contrastive.yaml} (75%)

diff --git a/lmms_eval/tasks/commute/commute-all.yaml b/lmms_eval/tasks/commute/commute-all-contrastive.yaml
similarity index 75%
rename from lmms_eval/tasks/commute/commute-all.yaml
rename to lmms_eval/tasks/commute/commute-all-contrastive.yaml
index 605f4be5e..80b1d60a0 100644
--- a/lmms_eval/tasks/commute/commute-all.yaml
+++ b/lmms_eval/tasks/commute/commute-all-contrastive.yaml
@@ -1,4 +1,4 @@
-group: commute-all
+group: commute-all-contrastive
 task:
   - commute-de
   - commute-ar
diff --git a/lmms_eval/tasks/commute/commute-ar.yaml b/lmms_eval/tasks/commute/commute-ar.yaml
index 9610f66b1..2185ead36 100644
--- a/lmms_eval/tasks/commute/commute-ar.yaml
+++ b/lmms_eval/tasks/commute/commute-ar.yaml
@@ -15,16 +15,16 @@ process_results: !function utils.commute_process_results
 
 doc_to_target: "answer"
 
-output_type: generate_until
-generation_kwargs:
-  max_new_tokens: 128
-  temperature: 0
-  do_sample: False
+output_type: loglikelihood
+# generation_kwargs:
+#   max_new_tokens: 128
+#   temperature: 0
+#   do_sample: False
 
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following text from English to Arabic by taking into account the image to resolve any ambiguities.\nEnglish: {source}\nArabic: "
+    pre_prompt: "Translate the following source text from English to Arabic:\n{source}"
 
 metric_list:
   - metric: results
diff --git a/lmms_eval/tasks/commute/commute-cs.yaml b/lmms_eval/tasks/commute/commute-cs.yaml
index e59d444c2..d659cb725 100644
--- a/lmms_eval/tasks/commute/commute-cs.yaml
+++ b/lmms_eval/tasks/commute/commute-cs.yaml
@@ -15,16 +15,16 @@ process_results: !function utils.commute_process_results
 
 doc_to_target: "answer"
 
-output_type: generate_until
-generation_kwargs:
-  max_new_tokens: 128
-  temperature: 0
-  do_sample: False
+output_type: loglikelihood
+# generation_kwargs:
+#   max_new_tokens: 128
+#   temperature: 0
+#   do_sample: False
 
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following text from English to Czech by taking into account the image to resolve any ambiguities.\nEnglish: {source}\nCzech: "
+    pre_prompt: "Translate the following source text from English to Czech:\n{source}"
 
 metric_list:
   - metric: results
diff --git a/lmms_eval/tasks/commute/commute-de.yaml b/lmms_eval/tasks/commute/commute-de.yaml
index 31cb645ab..f6f180dec 100644
--- a/lmms_eval/tasks/commute/commute-de.yaml
+++ b/lmms_eval/tasks/commute/commute-de.yaml
@@ -15,16 +15,16 @@ process_results: !function utils.commute_process_results
 
 doc_to_target: "answer"
 
-output_type: generate_until
-generation_kwargs:
-  max_new_tokens: 128
-  temperature: 0
-  do_sample: False
+output_type: loglikelihood
+# generation_kwargs:
+#   max_new_tokens: 128
+#   temperature: 0
+#   do_sample: False
 
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following text from English to German by taking into account the image to resolve any ambiguities.\nEnglish: {source}\nGerman: "
+    pre_prompt: "Translate the following source text from English to German:\n{source}"
 
 metric_list:
   - metric: results
diff --git a/lmms_eval/tasks/commute/commute-es.yaml b/lmms_eval/tasks/commute/commute-es.yaml
index 81cee1d0a..ae6efeae5 100644
--- a/lmms_eval/tasks/commute/commute-es.yaml
+++ b/lmms_eval/tasks/commute/commute-es.yaml
@@ -15,16 +15,16 @@ process_results: !function utils.commute_process_results
 
 doc_to_target: "answer"
 
-output_type: generate_until
-generation_kwargs:
-  max_new_tokens: 128
-  temperature: 0
-  do_sample: False
+output_type: loglikelihood
+# generation_kwargs:
+#   max_new_tokens: 128
+#   temperature: 0
+#   do_sample: False
 
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following text from English to Spanish by taking into account the image to resolve any ambiguities.\nEnglish: {source}\nSpanish: "
+    pre_prompt: "Translate the following source text from English to Spanish:\n{source}"
 
 metric_list:
   - metric: results
diff --git a/lmms_eval/tasks/commute/commute-fr.yaml b/lmms_eval/tasks/commute/commute-fr.yaml
index db2e2d68a..e302c9944 100644
--- a/lmms_eval/tasks/commute/commute-fr.yaml
+++ b/lmms_eval/tasks/commute/commute-fr.yaml
@@ -15,16 +15,16 @@ process_results: !function utils.commute_process_results
 
 doc_to_target: "answer"
 
-output_type: generate_until
-generation_kwargs:
-  max_new_tokens: 128
-  temperature: 0
-  do_sample: False
+output_type: loglikelihood
+# generation_kwargs:
+#   max_new_tokens: 128
+#   temperature: 0
+#   do_sample: False
 
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following text from English to French by taking into account the image to resolve any ambiguities.\nEnglish: {source}\nFrench: "
+    pre_prompt: "Translate the following source text from English to French:\n{source}"
 
 metric_list:
   - metric: results
diff --git a/lmms_eval/tasks/commute/commute-ru.yaml b/lmms_eval/tasks/commute/commute-ru.yaml
index 326962e8f..db9ad045f 100644
--- a/lmms_eval/tasks/commute/commute-ru.yaml
+++ b/lmms_eval/tasks/commute/commute-ru.yaml
@@ -15,16 +15,16 @@ process_results: !function utils.commute_process_results
 
 doc_to_target: "answer"
 
-output_type: generate_until
-generation_kwargs:
-  max_new_tokens: 128
-  temperature: 0
-  do_sample: False
+output_type: loglikelihood
+# generation_kwargs:
+#   max_new_tokens: 128
+#   temperature: 0
+#   do_sample: False
 
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following text from English to Russian by taking into account the image to resolve any ambiguities.\nEnglish: {source}\nRussian: "
+    pre_prompt: "Translate the following source text from English to Russian:\n{source}"
 
 metric_list:
   - metric: results
diff --git a/lmms_eval/tasks/commute/commute-zh.yaml b/lmms_eval/tasks/commute/commute-zh.yaml
index f98a63918..d8141d79b 100644
--- a/lmms_eval/tasks/commute/commute-zh.yaml
+++ b/lmms_eval/tasks/commute/commute-zh.yaml
@@ -15,16 +15,16 @@ process_results: !function utils.commute_process_results
 
 doc_to_target: "answer"
 
-output_type: generate_until
-generation_kwargs:
-  max_new_tokens: 128
-  temperature: 0
-  do_sample: False
+output_type: loglikelihood
+# generation_kwargs:
+#   max_new_tokens: 128
+#   temperature: 0
+#   do_sample: False
 
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following text from English to Chinese by taking into account the image to resolve any ambiguities.\nEnglish: {source}\nChinese: "
+    pre_prompt: "Translate the following source text from English to Chinese:\n{source}"
 
 metric_list:
   - metric: results
diff --git a/lmms_eval/tasks/commute/utils.py b/lmms_eval/tasks/commute/utils.py
index a67f60cac..938a0aec1 100644
--- a/lmms_eval/tasks/commute/utils.py
+++ b/lmms_eval/tasks/commute/utils.py
@@ -3,6 +3,7 @@
 from PIL import Image
 from io import BytesIO
 import numpy as np
+import copy
 
 from loguru import logger
 
@@ -17,55 +18,69 @@
 #     img_bytes = base64.b64decode(base64_string)
 #     return img_bytes
 
-
 def commute_process_docs(docs):
+
+    def add_output_to_docs(example):
+        return {
+            'output': example['correct_translation']
+        }
     # Process images in place
     # docs = docs.select(range(20)) # filter out some samples!
+    
+    samples = docs.to_list()
+    samples_duplicated = copy.deepcopy(samples)
+    new_samples = []
+    for sample in samples:
+        sample["output"] = sample["correct_translation"]
+        new_samples.append(sample)
+    for sample in samples_duplicated:
+        sample["output"] = sample["incorrect_translation"]
+        new_samples.append(sample)
+    docs = docs.from_list(new_samples)
     return docs
 
 def commute_doc_to_visual(doc):
-    image = doc["image"].convert('RGB')
+    image = Image.open(BytesIO(doc['image']['bytes']))
+    image = image.convert('RGB')
     return [image]
 
 
+
 def commute_doc_to_text(doc,lmms_eval_specific_kwargs=None ):
     source_txt = doc["source"]
     pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
     pre_prompt = pre_prompt.format(source=source_txt)
-    return f"{pre_prompt}"
+    input = pre_prompt+doc["output"]
+    return f"{input}"
 
 
 def commute_process_results(doc, results):
-    pred = results[0]
+    loglikelihood = results[0][0]
     target = doc["correct_translation"]
-    incorrect_translation = doc["incorrect_translation"]
     source = doc["source"]
-    return {"results": {"image_name": doc["image_name"], "source": source, "prediction": pred, "ground_truth": target, "incorrect_translation": incorrect_translation} }
-
-def compute_pairwise_accuracy(segments_scores_correct, segments_scores_incorrect):
-    pairwise_scores = []
-    for score_correct, score_incorrect in zip(segments_scores_correct, segments_scores_incorrect):
-        if score_correct > score_incorrect:
-            pairwise_scores.append(1)
-        else:
-            pairwise_scores.append(0)
-    pairwise_accuracy = sum(pairwise_scores) / len(pairwise_scores)
-    return pairwise_accuracy
+    return {"results": {"image_name": doc["image_name"], "source": source, "loglikelihood": loglikelihood, } }
 
 def commute_aggregate_results(results):
-    comet = RefCOMET(model="Unbabel/XCOMET-XL")
-    sources = [res["source"] for res in results]
-    hypotheses = [res["prediction"] for res in results]
-    references = [res["ground_truth"] for res in results]
-    incorrect_translations = [res["incorrect_translation"] for res in results]
-    comet.make_samples(sources, hypotheses, references)
-    segments_scores_correct = comet.evaluate(hypotheses, references, sources, gpus=1, batch_size=16).result["segments_scores"]
-    segments_scores_incorrect = comet.evaluate(hypotheses, incorrect_translations, sources, gpus=1, batch_size=16).result["segments_scores"]
-    pairwise_accuracy = compute_pairwise_accuracy(segments_scores_correct, segments_scores_incorrect)
-    results = {"pairwise_accuracy": pairwise_accuracy, 
-               "avg_correct_score": np.mean(segments_scores_correct), 
-               "avg_incorrect_score": np.mean(segments_scores_incorrect),
-            #    "segments_scores_correct": segments_scores_correct,
-            #    "segments_scores_incorrect": segments_scores_incorrect
-               }
+    # comet = RefCOMET(model="Unbabel/XCOMET-XL")
+    # sources = [res["source"] for res in results]
+    # hypotheses = [res["prediction"] for res in results]
+    # references = [res["ground_truth"] for res in results]
+    # incorrect_translations = [res["incorrect_translation"] for res in results]
+    # comet.make_samples(sources, hypotheses, references)
+    # segments_scores_correct = comet.evaluate(hypotheses, references, sources, gpus=1, batch_size=16).result["segments_scores"]
+    # segments_scores_incorrect = comet.evaluate(hypotheses, incorrect_translations, sources, gpus=1, batch_size=16).result["segments_scores"]
+    # pairwise_accuracy = compute_pairwise_accuracy(segments_scores_correct, segments_scores_incorrect)
+    
+    # Split results into two halves
+    half_idx = len(results) // 2
+    results_correct = results[:half_idx]
+    results_incorrect = results[half_idx:]
+
+    # Extract loglikelihoods from each half
+    loglikelihood_correct = [res["loglikelihood"] for res in results_correct]
+    loglikelihood_incorrect = [res["loglikelihood"] for res in results_incorrect]
+
+    accuracy = [1 for l_correct, l_incorrect in zip(loglikelihood_correct, loglikelihood_incorrect) if l_correct<l_incorrect ]
+    contrastive_accuracy = sum(accuracy)/len(loglikelihood_correct)
+    results = {"contrastive_accuracy": contrastive_accuracy}
     return results

From d38f6731504ebc900cc036ba980aed61f8821623 Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Thu, 20 Mar 2025 13:48:44 +0000
Subject: [PATCH 043/112] updated multi30k task with training prompt, support
 for multiple languages

---
 lmms_eval/tasks/multi30k/multi30k-all.yaml    |  5 +++
 lmms_eval/tasks/multi30k/multi30k-cs.yaml     | 36 ++++++++++++++++++
 lmms_eval/tasks/multi30k/multi30k-de.yaml     | 37 +++++++++++++++++++
 .../{multi30k.yaml => multi30k-fr.yaml}       |  8 ++--
 lmms_eval/tasks/multi30k/utils.py             | 19 ++++++++--
 5 files changed, 98 insertions(+), 7 deletions(-)
 create mode 100644 lmms_eval/tasks/multi30k/multi30k-all.yaml
 create mode 100644 lmms_eval/tasks/multi30k/multi30k-cs.yaml
 create mode 100644 lmms_eval/tasks/multi30k/multi30k-de.yaml
 rename lmms_eval/tasks/multi30k/{multi30k.yaml => multi30k-fr.yaml} (70%)

diff --git a/lmms_eval/tasks/multi30k/multi30k-all.yaml b/lmms_eval/tasks/multi30k/multi30k-all.yaml
new file mode 100644
index 000000000..2dc70780c
--- /dev/null
+++ b/lmms_eval/tasks/multi30k/multi30k-all.yaml
@@ -0,0 +1,5 @@
+group: multi30k-all
+task:
+  - multi30k-de
+  - multi30k-cs
+  - multi30k-fr
diff --git a/lmms_eval/tasks/multi30k/multi30k-cs.yaml b/lmms_eval/tasks/multi30k/multi30k-cs.yaml
new file mode 100644
index 000000000..503a68dfa
--- /dev/null
+++ b/lmms_eval/tasks/multi30k/multi30k-cs.yaml
@@ -0,0 +1,36 @@
+dataset_path: romrawinjp/multi30k
+task: "multi30k-cs"
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.multi30k_doc_to_visual
+doc_to_text: !function utils.multi30k_doc_to_text
+doc_to_target: "cs"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.multi30k_process_results_cs
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.multi30k_aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from English to Czech:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/multi30k/multi30k-de.yaml b/lmms_eval/tasks/multi30k/multi30k-de.yaml
new file mode 100644
index 000000000..300c0bd11
--- /dev/null
+++ b/lmms_eval/tasks/multi30k/multi30k-de.yaml
@@ -0,0 +1,37 @@
+dataset_path: romrawinjp/multi30k
+task: "multi30k-de"
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.multi30k_doc_to_visual
+doc_to_text: !function utils.multi30k_doc_to_text
+doc_to_target: "de"
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.multi30k_process_results_de
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.multi30k_aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    eval_model: "Unbabel/XCOMET-XXL"
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from English to German:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/multi30k/multi30k.yaml b/lmms_eval/tasks/multi30k/multi30k-fr.yaml
similarity index 70%
rename from lmms_eval/tasks/multi30k/multi30k.yaml
rename to lmms_eval/tasks/multi30k/multi30k-fr.yaml
index c48437cd5..36f3ad97c 100644
--- a/lmms_eval/tasks/multi30k/multi30k.yaml
+++ b/lmms_eval/tasks/multi30k/multi30k-fr.yaml
@@ -1,18 +1,18 @@
 dataset_path: romrawinjp/multi30k
-task: "multi30k"
+task: "multi30k-fr"
 test_split: test
 output_type: generate_until
 doc_to_visual: !function utils.multi30k_doc_to_visual
 doc_to_text: !function utils.multi30k_doc_to_text
 doc_to_target: "fr"
 generation_kwargs:
-  max_new_tokens: 16
+  max_new_tokens: 128
   temperature: 0
   top_p: 1.0
   num_beams: 1
   do_sample: false
 
-process_results: !function utils.multi30k_process_results
+process_results: !function utils.multi30k_process_results_fr
 
 metric_list:
   - metric: results
@@ -26,7 +26,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following caption from English to French by taking into account the image to resolve any ambiguities.\nEnglish: {source}\nFrench: "
+    pre_prompt: "Translate the following source text from English to French:\n{source}"
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
diff --git a/lmms_eval/tasks/multi30k/utils.py b/lmms_eval/tasks/multi30k/utils.py
index 3ccdb6300..db30abe18 100644
--- a/lmms_eval/tasks/multi30k/utils.py
+++ b/lmms_eval/tasks/multi30k/utils.py
@@ -2,7 +2,8 @@
 from loguru import logger as eval_logger
 from lmms_eval.tasks.multi30k.comet_utils.comet import RefCOMET
 
-
+# def multi30k_process_docs(docs):
+#     return docs
 
 def multi30k_doc_to_text(doc, lmms_eval_specific_kwargs):
     source_txt = doc["en"]
@@ -15,12 +16,24 @@ def multi30k_doc_to_visual(doc):
     return [doc["image"].convert("RGB")]
 
 
-def multi30k_process_results(doc, results):
+def multi30k_process_results_fr(doc, results):
     pred_answer = results[0]
     source = doc["en"]
     target = doc["fr"]
     return {"results":{"prediction":pred_answer,"ground_truth": target, "source":source}}
 
+def multi30k_process_results_de(doc, results):
+    pred_answer = results[0]
+    source = doc["en"]
+    target = doc["de"]
+    return {"results":{"prediction":pred_answer,"ground_truth": target, "source":source}}
+
+def multi30k_process_results_cs(doc, results):
+    pred_answer = results[0]
+    source = doc["en"]
+    target = doc["cs"]
+    return {"results":{"prediction":pred_answer,"ground_truth": target, "source":source}}
+
 def multi30k_aggregate_results(results):
     comet = RefCOMET(model="Unbabel/XCOMET-XL")
     sources = [res["source"] for res in results]
@@ -29,7 +42,7 @@ def multi30k_aggregate_results(results):
     comet.make_samples(sources, hypotheses, references)
     segments_scores_correct = comet.evaluate(hypotheses, references, sources, gpus=1, batch_size=16).result["segments_scores"]
     results = {
-               "avg_correct_score": np.mean(segments_scores_correct),
+               f"avg_XCOMET-XL_score": np.mean(segments_scores_correct),
                }
     return results
 

From cb97995177f3caeaa70b84b6b552b36f54df36c2 Mon Sep 17 00:00:00 2001
From: farajian <amin@unbabel.com>
Date: Thu, 20 Mar 2025 14:00:00 +0000
Subject: [PATCH 044/112] added custom system prompt to llava

---
 lmms_eval/models/llava.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lmms_eval/models/llava.py b/lmms_eval/models/llava.py
index 11ef8fcbf..3970bda8f 100755
--- a/lmms_eval/models/llava.py
+++ b/lmms_eval/models/llava.py
@@ -65,6 +65,7 @@ def __init__(
         tie_weights: bool = True,
         truncate_context=False,  # whether to truncate the context in generation, set it False for LLaVA-1.6
         customized_config=None,  # ends in json
+        system_prompt=None,
         **kwargs,
     ) -> None:
         super().__init__()
@@ -142,7 +143,7 @@ def __init__(
             self.model.to(self._device)
             self._rank = 0
             self._world_size = 1
-
+        self.system_prompt = system_prompt
     @property
     def config(self):
         # return the associated transformers.AutoConfig for the given pretrained model.
@@ -360,8 +361,8 @@ def _collate(x):
                     conv = conv_templates[self.conv_template].copy()
                 # if conv.system:
                 #     conv.system = self.add_system_prompt
-                # elif self.add_system_prompt:
-                #     conv.system = self.add_system_prompt
+                if self.system_prompt and conv.system:
+                    conv.system = self.system_prompt
                 conv.append_message(conv.roles[0], question)
                 conv.append_message(conv.roles[1], None)
                 prompt_question = conv.get_prompt()

From 846c5421c8ee00cb9fbb4b6d77e7b07a27fdb223 Mon Sep 17 00:00:00 2001
From: farajian <amin@unbabel.com>
Date: Thu, 20 Mar 2025 15:49:12 +0000
Subject: [PATCH 045/112] added the parser of results for cc-ocr task

---
 scripts/parse_results.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/scripts/parse_results.py b/scripts/parse_results.py
index 53abdc9f2..23fd5d742 100644
--- a/scripts/parse_results.py
+++ b/scripts/parse_results.py
@@ -60,6 +60,26 @@ def get_results_maxm(results, task_name, model_name):
         })
     return rows
 
+def get_results_cc_ocr(results, task_name, model_name):
+    rows = []
+    scores = {}
+    scores["de"] = float(results["cc-ocr-multi-lan-de"]["ocr_results,none"]["macro_f1_score"]) * 100
+    scores["es"] = float(results["cc-ocr-multi-lan-es"]["ocr_results,none"]["macro_f1_score"]) * 100
+    scores["fr"] = float(results["cc-ocr-multi-lan-fr"]["ocr_results,none"]["macro_f1_score"]) * 100
+    scores["it"] = float(results["cc-ocr-multi-lan-it"]["ocr_results,none"]["macro_f1_score"]) * 100
+    scores["ko"] = float(results["cc-ocr-multi-lan-ko"]["ocr_results,none"]["macro_f1_score"]) * 100
+    scores["pt"] = float(results["cc-ocr-multi-lan-pt"]["ocr_results,none"]["macro_f1_score"]) * 100
+    scores["ru"] = float(results["cc-ocr-multi-lan-ru"]["ocr_results,none"]["macro_f1_score"]) * 100
+    
+    for kk, vv in scores.items():
+        rows.append({
+            "model": model_name,
+            "task": task_name,
+            "language": kk,
+            "score": vv
+        })
+    return rows
+
 def get_results_mme(results, task_name, model_name):
     rows = []
     scores = {}
@@ -224,6 +244,7 @@ def process_results(data, task_name, model_name):
     # "xm100": get_results_xm100,
     "xmmmu": get_results_xmmmu,
     "ai2d": get_results_ai2d,
+    "cc-ocr-multi-lan": get_results_cc_ocr
 }
 
 

From 71384a208233939b301a077eeb83c66779f0c3f2 Mon Sep 17 00:00:00 2001
From: farajian <amin@unbabel.com>
Date: Thu, 20 Mar 2025 19:30:59 +0000
Subject: [PATCH 046/112] fixed the issue in alm_bench_doc_to_text

---
 lmms_eval/tasks/alm_bench/utils.py | 37 ++++++++++++++++++++++--------
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/lmms_eval/tasks/alm_bench/utils.py b/lmms_eval/tasks/alm_bench/utils.py
index 665f345be..839744980 100644
--- a/lmms_eval/tasks/alm_bench/utils.py
+++ b/lmms_eval/tasks/alm_bench/utils.py
@@ -1,6 +1,6 @@
 from PIL import Image
-
-
+import re
+import sys
 def exact_match(pred, target):
     if pred == target:
         return 1
@@ -11,6 +11,26 @@ def alm_bench_doc_to_visual(doc):
     image = (doc['file_name']).convert('RGB')
     return [image]
 
+def split_answer_options(text):
+    option_words = {
+        "english": "Options",
+        "dutch": "Opties",
+        "korean": "옵션",
+        "Chinese (Simplified)": "选项",
+        "Spanish": "Opciones",
+        "Italian": "opzioni",
+        "Russian": "Варианты",
+        "French": "choix",
+        "Portuguese": "Opções",
+        "German": "Optionen",
+    }
+    text = text.strip()
+    match = re.match(r"^(.*?)\s*\((?:Options|Opties|옵션|选项|Opciones|opzioni|Варианты|choix|Opções|Optionen):\s*(.*?)\)$", text, re.IGNORECASE)
+    if match:
+        true_answer = match.group(1).strip()
+        choices = re.sub("\s*,\s*", "\n", match.group(2))
+        return true_answer, choices
+    return None, None
 
 def alm_bench_doc_to_text(doc, lmms_eval_specific_kwargs):
     # Process MCQ question to extract answer, at the moment since we are not filtered the data set, I only process MCQ question types
@@ -20,13 +40,9 @@ def alm_bench_doc_to_text(doc, lmms_eval_specific_kwargs):
     post_prompt = lmms_eval_specific_kwargs["post_prompt"]
     pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
     if lmms_eval_specific_kwargs["prompt_format"] == "mcq" and doc["Question_Type"] == "Multiple Choice Questions":
-        choices = doc["Translated_Answer"].split("(Options: ")[1].rstrip(")").split(", ")
-        choices_str = "\n".join([f"{choice}" for choice in choices])
-        print(f"{pre_prompt}{question}\n{choices_str}{post_prompt}")
-        return f"{pre_prompt}{question}\n{choices_str}{post_prompt}"
-    # elif lmms_eval_specific_kwargs["prompt_format"] == "tf" and doc["Question_Type"] == "True False Question":
-    #     options = "\n".join(["true","false"])
-    #     return f"{pre_prompt}{question}{options}{post_prompt}"
+        _, choices = split_answer_options(doc["Translated_Answer"])
+        full_prompt = f"{pre_prompt} {question}\n{choices}{post_prompt}"
+        return full_prompt
     else:
         return "WRONG PROMPT. Write no answer. "
 
@@ -46,7 +62,8 @@ def alm_bench_process_results(doc, results):
 
 def alm_bench_doc_to_target(doc, model_specific_target_kwargs):
     if model_specific_target_kwargs == "mcq":
-        return doc["Translated_Answer"].split(" (Options: ")[0]
+        true_answer, _ = split_answer_options(doc["Translated_Answer"])
+        return true_answer
     # elif model_specific_target_kwargs == "tf":
     #     return doc["Translated_Answer"]
     else:

From 6a50ed8088b4cd933eea13dc9f99cdee58d9dae8 Mon Sep 17 00:00:00 2001
From: sonalsannigrahi <sannigrahisonal@gmail.com>
Date: Fri, 21 Mar 2025 06:12:29 +0000
Subject: [PATCH 047/112] update alm bench

---
 lmms_eval/tasks/alm_bench/alm_bench.yaml |  6 +--
 lmms_eval/tasks/alm_bench/utils.py       | 58 ++++++++++++++----------
 2 files changed, 37 insertions(+), 27 deletions(-)

diff --git a/lmms_eval/tasks/alm_bench/alm_bench.yaml b/lmms_eval/tasks/alm_bench/alm_bench.yaml
index 54d67a80e..09f728901 100644
--- a/lmms_eval/tasks/alm_bench/alm_bench.yaml
+++ b/lmms_eval/tasks/alm_bench/alm_bench.yaml
@@ -1,4 +1,4 @@
-dataset_path: MBZUAI/ALM-Bench
+dataset_path: sonalsannigrahi/alm-bench-mcq-subset
 task: "alm-bench"
 test_split: test
 output_type: generate_until
@@ -11,7 +11,7 @@ generation_kwargs:
   do_sample: False
 process_results: !function utils.alm_bench_process_results
 metric_list:
-  - metric: mcq
+  - metric: results
     aggregation: mean
     higher_is_better: true
 metadata:
@@ -21,7 +21,7 @@ model_specific_target_kwargs:
 lmms_eval_specific_kwargs:
   default:
     prompt_format: "mcq"
-    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    pre_prompt: "For the given Multiple Choice Question Answer below, answer from one of the options below. Strictly answer the choice only. No additional text."
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
diff --git a/lmms_eval/tasks/alm_bench/utils.py b/lmms_eval/tasks/alm_bench/utils.py
index 839744980..747cf8f65 100644
--- a/lmms_eval/tasks/alm_bench/utils.py
+++ b/lmms_eval/tasks/alm_bench/utils.py
@@ -1,6 +1,7 @@
 from PIL import Image
 import re
 import sys
+import numpy as np 
 def exact_match(pred, target):
     if pred == target:
         return 1
@@ -33,38 +34,47 @@ def split_answer_options(text):
     return None, None
 
 def alm_bench_doc_to_text(doc, lmms_eval_specific_kwargs):
-    # Process MCQ question to extract answer, at the moment since we are not filtered the data set, I only process MCQ question types
     question = doc["Translated_Question"]
     pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
     post_prompt = lmms_eval_specific_kwargs["post_prompt"]
     post_prompt = lmms_eval_specific_kwargs["post_prompt"]
-    pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
-    if lmms_eval_specific_kwargs["prompt_format"] == "mcq" and doc["Question_Type"] == "Multiple Choice Questions":
-        _, choices = split_answer_options(doc["Translated_Answer"])
-        full_prompt = f"{pre_prompt} {question}\n{choices}{post_prompt}"
-        return full_prompt
-    else:
-        return "WRONG PROMPT. Write no answer. "
+    _, choices = split_answer_options(doc["Translated_Answer"])
+    full_prompt = f"{pre_prompt} {question}\n{choices}{post_prompt}"
+    return full_prompt
 
 
 def alm_bench_process_results(doc, results):
     pred = results[0]
-    type = doc["Question_Type"]
-    return_dict = {"mcq": 0, "others": 0}
-    if type == "Multiple Choice Questions":
-        target = doc["Translated_Answer"].split(" (Options: ")[0]
-        match = exact_match(pred, target)
-        return_dict["mcq"] = match
-    else:
-        return_dict["others"] = 0
-    return return_dict
+    target, _ = split_answer_options(doc["Translated_Answer"])
+    match = exact_match(pred, target)
+    return {"results": match}
 
 
 def alm_bench_doc_to_target(doc, model_specific_target_kwargs):
-    if model_specific_target_kwargs == "mcq":
-        true_answer, _ = split_answer_options(doc["Translated_Answer"])
-        return true_answer
-    # elif model_specific_target_kwargs == "tf":
-    #     return doc["Translated_Answer"]
-    else:
-        return "NO ANSWER"
+    true_answer, _ = split_answer_options(doc["Translated_Answer"])
+    return true_answer
+
+#what if I report per language answers?
+
+## Here the results dict would have 'score' and 'language' keys
+
+# def alm_bench_aggregate_results(results):
+#     lang_dict = {
+#         "English": [],
+#         "Dutch": [],
+#         "Korean": [],
+#         "Chinese (Simplified)": [],
+#         "Spanish": [],
+#         "Italian": [],
+#         "Russian": [],
+#         "French": [],
+#         "Portuguese": [],
+#         "German": [],
+#     }
+#     for res in results:
+#         lang_dict[res["results"]["language"]].append(res["results"]["score"])
+
+#     for lang in lang_dict.keys():
+#         lang_dict[lang] = sum(lang_dict[lang])/len(lang_dict[lang])
+    
+#     return lang_dict
\ No newline at end of file

From 425110da100042553e199da5511b68892dad99e6 Mon Sep 17 00:00:00 2001
From: sonalsannigrahi <sannigrahisonal@gmail.com>
Date: Fri, 21 Mar 2025 14:38:37 +0000
Subject: [PATCH 048/112] update alm_bench prompt and parser

---
 lmms_eval/tasks/alm_bench/alm_bench.yaml |  2 +-
 scripts/parse_results.py                 | 40 +++++++++++++++++++++++-
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/lmms_eval/tasks/alm_bench/alm_bench.yaml b/lmms_eval/tasks/alm_bench/alm_bench.yaml
index 09f728901..38c088181 100644
--- a/lmms_eval/tasks/alm_bench/alm_bench.yaml
+++ b/lmms_eval/tasks/alm_bench/alm_bench.yaml
@@ -21,7 +21,7 @@ model_specific_target_kwargs:
 lmms_eval_specific_kwargs:
   default:
     prompt_format: "mcq"
-    pre_prompt: "For the given Multiple Choice Question Answer below, answer from one of the options below. Strictly answer the choice only. No additional text."
+    pre_prompt: "For the given the Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
diff --git a/scripts/parse_results.py b/scripts/parse_results.py
index 23fd5d742..445eb18a2 100644
--- a/scripts/parse_results.py
+++ b/scripts/parse_results.py
@@ -225,6 +225,42 @@ def get_results_ai2d(results, task_name, model_name):
         })
     return rows
 
+def get_results_multi30k(results, task_name, model_name):
+    rows = []
+    scores = {}
+    scores["de"] = float(results["multi30k-de"]["results,none"]["avg_XCOMET-XL_score"]) * 100
+    scores["fr"] = float(results["multi30k-fr"]["results,none"]["avg_XCOMET-XL_score"]) * 100
+    scores["cs"] = float(results["multi30k-cs"]["results,none"]["avg_XCOMET-XL_score"]) * 100
+    
+    for kk, vv in scores.items():
+        rows.append({
+            "model": model_name,
+            "task": task_name,
+            "language": kk,
+            "score": vv
+        })
+    return rows    
+
+def get_results_commute(results, task_name, model_name):
+    rows = []
+    scores = {}
+    scores["ar"] = float(results["commute-ar"]["results,none"]["contrastive_accuracy"]) * 100
+    scores["cs"] = float(results["commute-cs"]["results,none"]["contrastive_accuracy"]) * 100
+    scores["de"] = float(results["commute-de"]["results,none"]["contrastive_accuracy"]) * 100
+    scores["es"] = float(results["commute-es"]["results,none"]["contrastive_accuracy"]) * 100
+    scores["fr"] = float(results["commute-fr"]["results,none"]["contrastive_accuracy"]) * 100
+    scores["ru"] = float(results["commute-ru"]["results,none"]["contrastive_accuracy"]) * 100
+    scores["zh"] = float(results["commute-zh"]["results,none"]["contrastive_accuracy"]) * 100
+    
+    for kk, vv in scores.items():
+        rows.append({
+            "model": model_name,
+            "task": task_name,
+            "language": kk,
+            "score": vv
+        })
+    return rows   
+
 def process_results(data, task_name, model_name):
     results = data["results"]
     results = result_functions[task_name](results, task_name, model_name)
@@ -244,7 +280,9 @@ def process_results(data, task_name, model_name):
     # "xm100": get_results_xm100,
     "xmmmu": get_results_xmmmu,
     "ai2d": get_results_ai2d,
-    "cc-ocr-multi-lan": get_results_cc_ocr
+    "cc-ocr-multi-lan": get_results_cc_ocr,
+    "multi30k-all": get_results_multi30k,
+    "commute-all-contrastive": get_results_commute
 }
 
 

From b798548f5843d01122c568d12e4e8fc79a7eaec6 Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Fri, 21 Mar 2025 14:44:24 +0000
Subject: [PATCH 049/112] updated cc-ocr

---
 .../tasks/cc_ocr/cc-ocr-multi-lan-de.yaml     |  16 +--
 .../tasks/cc_ocr/cc-ocr-multi-lan-es.yaml     |  16 +--
 .../tasks/cc_ocr/cc-ocr-multi-lan-fr.yaml     |  15 +--
 .../tasks/cc_ocr/cc-ocr-multi-lan-it.yaml     |  16 +--
 .../tasks/cc_ocr/cc-ocr-multi-lan-ko.yaml     |  15 +--
 .../tasks/cc_ocr/cc-ocr-multi-lan-pt.yaml     |  15 +--
 .../tasks/cc_ocr/cc-ocr-multi-lan-ru.yaml     |  15 +--
 lmms_eval/tasks/cc_ocr/utils.py               | 111 ++++++++++++++++--
 8 files changed, 128 insertions(+), 91 deletions(-)

diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml
index 538861744..dfec86468 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-de.yaml
@@ -7,17 +7,9 @@ test_split: test
 
 dataset_kwargs:
   token: True
-  data_files:
-    test: [ "multi_lan_ocr/German/*.tsv", 
-    #"multi_lan_ocr/German/*.tsv", 
-    # "multi_lan_ocr/Italian/*.tsv" ,
-    # "multi_lan_ocr/Korean/*.tsv",
-     #"multi_lan_ocr/Portuguese/*.tsv",
-     # "multi_lan_ocr/Russian/*.tsv",
-     # "multi_lan_ocr/Spanish/*.tsv"
-     ] 
-
-process_docs:  !function utils.ccocr_process_docs
+
+
+process_docs:  !function utils.ccocr_process_docs_de
 doc_to_visual: !function utils.ccocr_doc_to_visual
 doc_to_text: !function utils.ccocr_doc_to_text
 process_results: !function utils.ccocr_process_results
@@ -25,7 +17,7 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 128
+  max_new_tokens: 1024
   temperature: 0
   do_sample: False
 
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-es.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-es.yaml
index 64c5db5de..03b05410c 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-es.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-es.yaml
@@ -7,17 +7,9 @@ test_split: test
 
 dataset_kwargs:
   token: True
-  data_files:
-    test: [ #"multi_lan_ocr/German/*.tsv", 
-    # "multi_lan_ocr/French/*.tsv", 
-    # "multi_lan_ocr/Italian/*.tsv" ,
-    # "multi_lan_ocr/Korean/*.tsv",
-    #  "multi_lan_ocr/Portuguese/*.tsv",
-    #  "multi_lan_ocr/Russian/*.tsv",
-     "multi_lan_ocr/Spanish/*.tsv"
-     ] 
-
-process_docs:  !function utils.ccocr_process_docs
+ 
+
+process_docs:  !function utils.ccocr_process_docs_es
 doc_to_visual: !function utils.ccocr_doc_to_visual
 doc_to_text: !function utils.ccocr_doc_to_text
 process_results: !function utils.ccocr_process_results
@@ -25,7 +17,7 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 128
+  max_new_tokens: 1024
   temperature: 0
   do_sample: False
 
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-fr.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-fr.yaml
index 0ebe05ee4..9e2b758b5 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-fr.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-fr.yaml
@@ -7,17 +7,8 @@ test_split: test
 
 dataset_kwargs:
   token: True
-  data_files:
-    test: [ #"multi_lan_ocr/German/*.tsv", 
-    "multi_lan_ocr/French/*.tsv", 
-    # "multi_lan_ocr/Italian/*.tsv" ,
-    # "multi_lan_ocr/Korean/*.tsv",
-     #"multi_lan_ocr/Portuguese/*.tsv",
-     # "multi_lan_ocr/Russian/*.tsv",
-     # "multi_lan_ocr/Spanish/*.tsv"
-     ] 
-
-process_docs:  !function utils.ccocr_process_docs
+
+process_docs:  !function utils.ccocr_process_docs_fr
 doc_to_visual: !function utils.ccocr_doc_to_visual
 doc_to_text: !function utils.ccocr_doc_to_text
 process_results: !function utils.ccocr_process_results
@@ -25,7 +16,7 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 128
+  max_new_tokens: 1024
   temperature: 0
   do_sample: False
 
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-it.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-it.yaml
index 9db9bba44..2a379588d 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-it.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-it.yaml
@@ -7,17 +7,9 @@ test_split: test
 
 dataset_kwargs:
   token: True
-  data_files:
-    test: [ #"multi_lan_ocr/German/*.tsv", 
-    # "multi_lan_ocr/French/*.tsv", 
-    "multi_lan_ocr/Italian/*.tsv" ,
-    # "multi_lan_ocr/Korean/*.tsv",
-    #  "multi_lan_ocr/Portuguese/*.tsv",
-     # "multi_lan_ocr/Russian/*.tsv",
-     # "multi_lan_ocr/Spanish/*.tsv"
-     ] 
-
-process_docs:  !function utils.ccocr_process_docs
+
+
+process_docs:  !function utils.ccocr_process_docs_it
 doc_to_visual: !function utils.ccocr_doc_to_visual
 doc_to_text: !function utils.ccocr_doc_to_text
 process_results: !function utils.ccocr_process_results
@@ -25,7 +17,7 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 128
+  max_new_tokens: 1024
   temperature: 0
   do_sample: False
 
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ko.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ko.yaml
index a5fedf42e..c28ae5f9c 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ko.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ko.yaml
@@ -7,17 +7,8 @@ test_split: test
 
 dataset_kwargs:
   token: True
-  data_files:
-    test: [ #"multi_lan_ocr/German/*.tsv", 
-    # "multi_lan_ocr/French/*.tsv", 
-    # "multi_lan_ocr/Italian/*.tsv" ,
-    "multi_lan_ocr/Korean/*.tsv",
-    #  "multi_lan_ocr/Portuguese/*.tsv",
-    #  "multi_lan_ocr/Russian/*.tsv",
-     # "multi_lan_ocr/Spanish/*.tsv"
-     ] 
-
-process_docs:  !function utils.ccocr_process_docs
+
+process_docs:  !function utils.ccocr_process_docs_ko
 doc_to_visual: !function utils.ccocr_doc_to_visual
 doc_to_text: !function utils.ccocr_doc_to_text
 process_results: !function utils.ccocr_process_results
@@ -25,7 +16,7 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 128
+  max_new_tokens: 1024
   temperature: 0
   do_sample: False
 
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-pt.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-pt.yaml
index 91d897ab6..8f235b471 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-pt.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-pt.yaml
@@ -7,17 +7,8 @@ test_split: test
 
 dataset_kwargs:
   token: True
-  data_files:
-    test: [ #"multi_lan_ocr/German/*.tsv", 
-    # "multi_lan_ocr/French/*.tsv", 
-    # "multi_lan_ocr/Italian/*.tsv" ,
-    # "multi_lan_ocr/Korean/*.tsv",
-     "multi_lan_ocr/Portuguese/*.tsv",
-     # "multi_lan_ocr/Russian/*.tsv",
-     # "multi_lan_ocr/Spanish/*.tsv"
-     ] 
-
-process_docs:  !function utils.ccocr_process_docs
+
+process_docs:  !function utils.ccocr_process_docs_pt
 doc_to_visual: !function utils.ccocr_doc_to_visual
 doc_to_text: !function utils.ccocr_doc_to_text
 process_results: !function utils.ccocr_process_results
@@ -25,7 +16,7 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 128
+  max_new_tokens: 1024
   temperature: 0
   do_sample: False
 
diff --git a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ru.yaml b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ru.yaml
index a39630f83..4e263649e 100644
--- a/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ru.yaml
+++ b/lmms_eval/tasks/cc_ocr/cc-ocr-multi-lan-ru.yaml
@@ -7,17 +7,8 @@ test_split: test
 
 dataset_kwargs:
   token: True
-  data_files:
-    test: [ #"multi_lan_ocr/German/*.tsv", 
-    # "multi_lan_ocr/French/*.tsv", 
-    # "multi_lan_ocr/Italian/*.tsv" ,
-    # "multi_lan_ocr/Korean/*.tsv",
-    #  "multi_lan_ocr/Portuguese/*.tsv",
-     "multi_lan_ocr/Russian/*.tsv",
-     # "multi_lan_ocr/Spanish/*.tsv"
-     ] 
-
-process_docs:  !function utils.ccocr_process_docs
+
+process_docs:  !function utils.ccocr_process_docs_ru
 doc_to_visual: !function utils.ccocr_doc_to_visual
 doc_to_text: !function utils.ccocr_doc_to_text
 process_results: !function utils.ccocr_process_results
@@ -25,7 +16,7 @@ doc_to_target: "answer"
 
 output_type: generate_until
 generation_kwargs:
-  max_new_tokens: 128
+  max_new_tokens: 1024
   temperature: 0
   do_sample: False
 
diff --git a/lmms_eval/tasks/cc_ocr/utils.py b/lmms_eval/tasks/cc_ocr/utils.py
index cfb50bc77..9bb9e9aaf 100644
--- a/lmms_eval/tasks/cc_ocr/utils.py
+++ b/lmms_eval/tasks/cc_ocr/utils.py
@@ -17,7 +17,7 @@ def base64_to_bytes(base64_string):
     return img_bytes
 
 
-def ccocr_process_docs(docs):
+def ccocr_process_docs_de(docs):
     """
     Process documents by converting base64 images to PIL Images
     
@@ -26,16 +26,113 @@ def ccocr_process_docs(docs):
     Returns:
         Dataset with converted PIL images
     """
-    logger.info(f"Converting base64 images to PIL Images...")
+    logger.info(f"Selecting specific language and converting base64 images to PIL Images...")
     # Process images in place
     # docs = docs.select(range(20)) # filter out some samples!
-    docs = docs.map(
-        lambda doc: {
-            'image': base64_to_bytes(doc['image'])
-        }
-    )
+    filter_docs = docs.filter(lambda doc: doc['split'] == 'German')
+    docs = filter_docs.map( lambda doc: {'image': base64_to_bytes(doc['image'])})
     return docs
 
+def ccocr_process_docs_es(docs):
+    """
+    Process documents by converting base64 images to PIL Images
+    
+    Args:
+        docs: Dataset object containing documents with base64 encoded images
+    Returns:
+        Dataset with converted PIL images
+    """
+    logger.info(f"Selecting specific language and converting base64 images to PIL Images...")
+    # Process images in place
+    # docs = docs.select(range(20)) # filter out some samples!
+    filter_docs = docs.filter(lambda doc: doc['split'] == 'Spanish')
+    docs = filter_docs.map( lambda doc: {'image': base64_to_bytes(doc['image'])})
+    return docs
+
+
+def ccocr_process_docs_fr(docs):
+    """
+    Process documents by converting base64 images to PIL Images
+    
+    Args:
+        docs: Dataset object containing documents with base64 encoded images
+    Returns:
+        Dataset with converted PIL images
+    """
+    logger.info(f"Selecting specific language and converting base64 images to PIL Images...")
+    # Process images in place
+    # docs = docs.select(range(20)) # filter out some samples!
+    filter_docs = docs.filter(lambda doc: doc['split'] == 'French')
+    docs = filter_docs.map( lambda doc: {'image': base64_to_bytes(doc['image'])})
+    return docs
+
+def ccocr_process_docs_it(docs):
+    """
+    Process documents by converting base64 images to PIL Images
+    
+    Args:
+        docs: Dataset object containing documents with base64 encoded images
+    Returns:
+        Dataset with converted PIL images
+    """
+    logger.info(f"Selecting specific language and converting base64 images to PIL Images...")
+    # Process images in place
+    # docs = docs.select(range(20)) # filter out some samples!
+    filter_docs = docs.filter(lambda doc: doc['split'] == 'Italian')
+    docs = filter_docs.map( lambda doc: {'image': base64_to_bytes(doc['image'])})
+    return docs
+
+
+def ccocr_process_docs_ko(docs):
+    """
+    Process documents by converting base64 images to PIL Images
+    
+    Args:
+        docs: Dataset object containing documents with base64 encoded images
+    Returns:
+        Dataset with converted PIL images
+    """
+    logger.info(f"Selecting specific language and converting base64 images to PIL Images...")
+    # Process images in place
+    # docs = docs.select(range(20)) # filter out some samples!
+    filter_docs = docs.filter(lambda doc: doc['split'] == 'Korean')
+    docs = filter_docs.map( lambda doc: {'image': base64_to_bytes(doc['image'])})
+    return docs
+
+def ccocr_process_docs_pt(docs):
+    """
+    Process documents by converting base64 images to PIL Images
+    
+    Args:
+        docs: Dataset object containing documents with base64 encoded images
+    Returns:
+        Dataset with converted PIL images
+    """
+    logger.info(f"Selecting specific language and converting base64 images to PIL Images...")
+    # Process images in place
+    # docs = docs.select(range(20)) # filter out some samples!
+    filter_docs = docs.filter(lambda doc: doc['split'] == 'Portuguese')
+    docs = filter_docs.map( lambda doc: {'image': base64_to_bytes(doc['image'])})
+    return docs
+
+
+def ccocr_process_docs_ru(docs):
+    """
+    Process documents by converting base64 images to PIL Images
+    
+    Args:
+        docs: Dataset object containing documents with base64 encoded images
+    Returns:
+        Dataset with converted PIL images
+    """
+    logger.info(f"Selecting specific language and converting base64 images to PIL Images...")
+    # Process images in place
+    # docs = docs.select(range(20)) # filter out some samples!
+    filter_docs = docs.filter(lambda doc: doc['split'] == 'Russian')
+    docs = filter_docs.map( lambda doc: {'image': base64_to_bytes(doc['image'])})
+    return docs
+
+
 def ccocr_doc_to_visual(doc):
     image = Image.open(BytesIO(doc["image"]))
     image = image.convert('RGB')

From 96635c203246a29356496da9f3f24f31c7ac0a24 Mon Sep 17 00:00:00 2001
From: Sonal Sannigrahi <sannigrahisonal@gmail.com>
Date: Fri, 21 Mar 2025 16:28:41 +0000
Subject: [PATCH 050/112] alm_bench per language

---
 lmms_eval/tasks/alm_bench/alm_bench-all.yaml | 11 ++++++++
 lmms_eval/tasks/alm_bench/alm_bench-de.yaml  | 27 +++++++++++++++++++
 lmms_eval/tasks/alm_bench/alm_bench-en.yaml  | 27 +++++++++++++++++++
 lmms_eval/tasks/alm_bench/alm_bench-es.yaml  | 27 +++++++++++++++++++
 lmms_eval/tasks/alm_bench/alm_bench-fr.yaml  | 27 +++++++++++++++++++
 lmms_eval/tasks/alm_bench/alm_bench-it.yaml  | 27 +++++++++++++++++++
 lmms_eval/tasks/alm_bench/alm_bench-ko.yaml  | 27 +++++++++++++++++++
 lmms_eval/tasks/alm_bench/alm_bench-nl.yaml  | 27 +++++++++++++++++++
 lmms_eval/tasks/alm_bench/alm_bench-pt.yaml  | 27 +++++++++++++++++++
 lmms_eval/tasks/alm_bench/alm_bench-ru.yaml  | 27 +++++++++++++++++++
 lmms_eval/tasks/alm_bench/utils.py           | 28 ++------------------
 11 files changed, 256 insertions(+), 26 deletions(-)
 create mode 100644 lmms_eval/tasks/alm_bench/alm_bench-all.yaml
 create mode 100644 lmms_eval/tasks/alm_bench/alm_bench-de.yaml
 create mode 100644 lmms_eval/tasks/alm_bench/alm_bench-en.yaml
 create mode 100644 lmms_eval/tasks/alm_bench/alm_bench-es.yaml
 create mode 100644 lmms_eval/tasks/alm_bench/alm_bench-fr.yaml
 create mode 100644 lmms_eval/tasks/alm_bench/alm_bench-it.yaml
 create mode 100644 lmms_eval/tasks/alm_bench/alm_bench-ko.yaml
 create mode 100644 lmms_eval/tasks/alm_bench/alm_bench-nl.yaml
 create mode 100644 lmms_eval/tasks/alm_bench/alm_bench-pt.yaml
 create mode 100644 lmms_eval/tasks/alm_bench/alm_bench-ru.yaml

diff --git a/lmms_eval/tasks/alm_bench/alm_bench-all.yaml b/lmms_eval/tasks/alm_bench/alm_bench-all.yaml
new file mode 100644
index 000000000..d15f41fa2
--- /dev/null
+++ b/lmms_eval/tasks/alm_bench/alm_bench-all.yaml
@@ -0,0 +1,11 @@
+group: alm_bench-all
+task:
+  - alm_bench-de
+  - alm_bench-es
+  - alm_bench-fr
+  - alm_bench-it
+  - alm_bench-ko
+  - alm_bench-nl
+  - alm_bench-pt
+  - alm_bench-ru
+  - alm_bench-en
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm_bench-de.yaml b/lmms_eval/tasks/alm_bench/alm_bench-de.yaml
new file mode 100644
index 000000000..3348e6e36
--- /dev/null
+++ b/lmms_eval/tasks/alm_bench/alm_bench-de.yaml
@@ -0,0 +1,27 @@
+dataset_path: sonalsannigrahi/alm-bench-lang-split
+task: "alm-bench-de"
+test_split: German
+output_type: generate_until
+doc_to_visual: !function utils.alm_bench_doc_to_visual
+doc_to_text: !function utils.alm_bench_doc_to_text
+doc_to_target: !function utils.alm_bench_doc_to_target
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  do_sample: False
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  - version: 0.0
+model_specific_target_kwargs:
+  default: "mcq"
+lmms_eval_specific_kwargs:
+  default:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm_bench-en.yaml b/lmms_eval/tasks/alm_bench/alm_bench-en.yaml
new file mode 100644
index 000000000..e87c05b45
--- /dev/null
+++ b/lmms_eval/tasks/alm_bench/alm_bench-en.yaml
@@ -0,0 +1,27 @@
+dataset_path: sonalsannigrahi/alm-bench-lang-split
+task: "alm-bench-en"
+test_split: English
+output_type: generate_until
+doc_to_visual: !function utils.alm_bench_doc_to_visual
+doc_to_text: !function utils.alm_bench_doc_to_text
+doc_to_target: !function utils.alm_bench_doc_to_target
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  do_sample: False
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  - version: 0.0
+model_specific_target_kwargs:
+  default: "mcq"
+lmms_eval_specific_kwargs:
+  default:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm_bench-es.yaml b/lmms_eval/tasks/alm_bench/alm_bench-es.yaml
new file mode 100644
index 000000000..b3dd093a5
--- /dev/null
+++ b/lmms_eval/tasks/alm_bench/alm_bench-es.yaml
@@ -0,0 +1,27 @@
+dataset_path: sonalsannigrahi/alm-bench-lang-split
+task: "alm-bench-es"
+test_split: Spanish
+output_type: generate_until
+doc_to_visual: !function utils.alm_bench_doc_to_visual
+doc_to_text: !function utils.alm_bench_doc_to_text
+doc_to_target: !function utils.alm_bench_doc_to_target
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  do_sample: False
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  - version: 0.0
+model_specific_target_kwargs:
+  default: "mcq"
+lmms_eval_specific_kwargs:
+  default:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm_bench-fr.yaml b/lmms_eval/tasks/alm_bench/alm_bench-fr.yaml
new file mode 100644
index 000000000..816e737d4
--- /dev/null
+++ b/lmms_eval/tasks/alm_bench/alm_bench-fr.yaml
@@ -0,0 +1,27 @@
+dataset_path: sonalsannigrahi/alm-bench-lang-split
+task: "alm-bench-fr"
+test_split: French
+output_type: generate_until
+doc_to_visual: !function utils.alm_bench_doc_to_visual
+doc_to_text: !function utils.alm_bench_doc_to_text
+doc_to_target: !function utils.alm_bench_doc_to_target
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  do_sample: False
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  - version: 0.0
+model_specific_target_kwargs:
+  default: "mcq"
+lmms_eval_specific_kwargs:
+  default:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm_bench-it.yaml b/lmms_eval/tasks/alm_bench/alm_bench-it.yaml
new file mode 100644
index 000000000..e48c1c3dc
--- /dev/null
+++ b/lmms_eval/tasks/alm_bench/alm_bench-it.yaml
@@ -0,0 +1,27 @@
+dataset_path: sonalsannigrahi/alm-bench-lang-split
+task: "alm-bench-it"
+test_split: Italian
+output_type: generate_until
+doc_to_visual: !function utils.alm_bench_doc_to_visual
+doc_to_text: !function utils.alm_bench_doc_to_text
+doc_to_target: !function utils.alm_bench_doc_to_target
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  do_sample: False
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  - version: 0.0
+model_specific_target_kwargs:
+  default: "mcq"
+lmms_eval_specific_kwargs:
+  default:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm_bench-ko.yaml b/lmms_eval/tasks/alm_bench/alm_bench-ko.yaml
new file mode 100644
index 000000000..9ebf5003c
--- /dev/null
+++ b/lmms_eval/tasks/alm_bench/alm_bench-ko.yaml
@@ -0,0 +1,27 @@
+dataset_path: sonalsannigrahi/alm-bench-lang-split
+task: "alm-bench-ko"
+test_split: Korean
+output_type: generate_until
+doc_to_visual: !function utils.alm_bench_doc_to_visual
+doc_to_text: !function utils.alm_bench_doc_to_text
+doc_to_target: !function utils.alm_bench_doc_to_target
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  do_sample: False
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  - version: 0.0
+model_specific_target_kwargs:
+  default: "mcq"
+lmms_eval_specific_kwargs:
+  default:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm_bench-nl.yaml b/lmms_eval/tasks/alm_bench/alm_bench-nl.yaml
new file mode 100644
index 000000000..cca54bad9
--- /dev/null
+++ b/lmms_eval/tasks/alm_bench/alm_bench-nl.yaml
@@ -0,0 +1,27 @@
+dataset_path: sonalsannigrahi/alm-bench-lang-split
+task: "alm-bench-nl"
+test_split: Dutch
+output_type: generate_until
+doc_to_visual: !function utils.alm_bench_doc_to_visual
+doc_to_text: !function utils.alm_bench_doc_to_text
+doc_to_target: !function utils.alm_bench_doc_to_target
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  do_sample: False
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  - version: 0.0
+model_specific_target_kwargs:
+  default: "mcq"
+lmms_eval_specific_kwargs:
+  default:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm_bench-pt.yaml b/lmms_eval/tasks/alm_bench/alm_bench-pt.yaml
new file mode 100644
index 000000000..e942f3926
--- /dev/null
+++ b/lmms_eval/tasks/alm_bench/alm_bench-pt.yaml
@@ -0,0 +1,27 @@
+dataset_path: sonalsannigrahi/alm-bench-lang-split
+task: "alm-bench-pt"
+test_split: Portuguese
+output_type: generate_until
+doc_to_visual: !function utils.alm_bench_doc_to_visual
+doc_to_text: !function utils.alm_bench_doc_to_text
+doc_to_target: !function utils.alm_bench_doc_to_target
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  do_sample: False
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  - version: 0.0
+model_specific_target_kwargs:
+  default: "mcq"
+lmms_eval_specific_kwargs:
+  default:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm_bench-ru.yaml b/lmms_eval/tasks/alm_bench/alm_bench-ru.yaml
new file mode 100644
index 000000000..c26f2c19b
--- /dev/null
+++ b/lmms_eval/tasks/alm_bench/alm_bench-ru.yaml
@@ -0,0 +1,27 @@
+dataset_path: sonalsannigrahi/alm-bench-lang-split
+task: "alm-bench-ru"
+test_split: Russian
+output_type: generate_until
+doc_to_visual: !function utils.alm_bench_doc_to_visual
+doc_to_text: !function utils.alm_bench_doc_to_text
+doc_to_target: !function utils.alm_bench_doc_to_target
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  do_sample: False
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  - version: 0.0
+model_specific_target_kwargs:
+  default: "mcq"
+lmms_eval_specific_kwargs:
+  default:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/utils.py b/lmms_eval/tasks/alm_bench/utils.py
index 747cf8f65..9ebc00749 100644
--- a/lmms_eval/tasks/alm_bench/utils.py
+++ b/lmms_eval/tasks/alm_bench/utils.py
@@ -2,6 +2,7 @@
 import re
 import sys
 import numpy as np 
+
 def exact_match(pred, target):
     if pred == target:
         return 1
@@ -52,29 +53,4 @@ def alm_bench_process_results(doc, results):
 
 def alm_bench_doc_to_target(doc, model_specific_target_kwargs):
     true_answer, _ = split_answer_options(doc["Translated_Answer"])
-    return true_answer
-
-#what if I report per language answers?
-
-## Here the results dict would have 'score' and 'language' keys
-
-# def alm_bench_aggregate_results(results):
-#     lang_dict = {
-#         "English": [],
-#         "Dutch": [],
-#         "Korean": [],
-#         "Chinese (Simplified)": [],
-#         "Spanish": [],
-#         "Italian": [],
-#         "Russian": [],
-#         "French": [],
-#         "Portuguese": [],
-#         "German": [],
-#     }
-#     for res in results:
-#         lang_dict[res["results"]["language"]].append(res["results"]["score"])
-
-#     for lang in lang_dict.keys():
-#         lang_dict[lang] = sum(lang_dict[lang])/len(lang_dict[lang])
-    
-#     return lang_dict
\ No newline at end of file
+    return true_answer
\ No newline at end of file

From 7a9c2f9b47a84d4e596e334d4cb0ee484141a1c1 Mon Sep 17 00:00:00 2001
From: Sonal Sannigrahi <sannigrahisonal@gmail.com>
Date: Fri, 21 Mar 2025 16:29:59 +0000
Subject: [PATCH 051/112] parser for alm bench

---
 scripts/parse_results.py | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/scripts/parse_results.py b/scripts/parse_results.py
index 445eb18a2..bec388a21 100644
--- a/scripts/parse_results.py
+++ b/scripts/parse_results.py
@@ -261,6 +261,28 @@ def get_results_commute(results, task_name, model_name):
         })
     return rows   
 
+def get_results_alm_bench(results, task_name, model_name):
+    rows = []
+    scores = {}
+    scores["de"] = float(results["alm_bench-de"]["exact_match,none"]) * 100
+    scores["es"] = float(results["alm_bench-es"]["exact_match,none"]) * 100
+    scores["fr"] = float(results["alm_bench-fr"]["exact_match,none"]) * 100
+    scores["it"] = float(results["alm_bench-it"]["exact_match,none"]) * 100
+    scores["ko"] = float(results["alm_bench-ko"]["exact_match,none"]) * 100
+    scores["nl"] = float(results["alm_bench-nl"]["exact_match,none"]) * 100
+    scores["pt"] = float(results["alm_bench-pt"]["exact_match,none"]) * 100
+    scores["ru"] = float(results["alm_bench-ru"]["exact_match,none"]) * 100
+    scores["en"] = float(results["alm_bench-en"]["exact_match,none"]) * 100
+    
+    for kk, vv in scores.items():
+        rows.append({
+            "model": model_name,
+            "task": task_name,
+            "language": kk,
+            "score": vv
+        })
+    return rows   
+
 def process_results(data, task_name, model_name):
     results = data["results"]
     results = result_functions[task_name](results, task_name, model_name)
@@ -282,7 +304,8 @@ def process_results(data, task_name, model_name):
     "ai2d": get_results_ai2d,
     "cc-ocr-multi-lan": get_results_cc_ocr,
     "multi30k-all": get_results_multi30k,
-    "commute-all-contrastive": get_results_commute
+    "commute-all-contrastive": get_results_commute,
+    "alm_bench-all": get_results_alm_bench
 }
 
 

From c2c02988076551dcd16cf6e9f63c3de65f69c9b0 Mon Sep 17 00:00:00 2001
From: farajian <amin@unbabel.com>
Date: Fri, 21 Mar 2025 16:34:54 +0000
Subject: [PATCH 052/112] updated the settings of ai2d. we might want to roll
 them back

---
 lmms_eval/tasks/ai2d/ai2d.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lmms_eval/tasks/ai2d/ai2d.yaml b/lmms_eval/tasks/ai2d/ai2d.yaml
index 7cf9cb013..3b4a33e02 100755
--- a/lmms_eval/tasks/ai2d/ai2d.yaml
+++ b/lmms_eval/tasks/ai2d/ai2d.yaml
@@ -12,7 +12,8 @@ lmms_eval_specific_kwargs:
   default:
     prompt_format: mcq
     pre_prompt: ""
-    post_prompt: "\nAnswer with the option's letter from the given choices directly."
+    post_prompt: ""
+    # post_prompt: "\nAnswer with the option's letter from the given choices directly."
   gpt4v: 
     prompt_format: mcq
     pre_prompt: ""
@@ -31,7 +32,7 @@ model_specific_target_kwargs:
   qwen_vl: "qa"
 
 generation_kwargs:
-  max_new_tokens: 16
+  max_new_tokens: 8192
   temperature: 0
   do_sample: False
 

From 410f349834737d4fe410923cea1bdd66f928088b Mon Sep 17 00:00:00 2001
From: Sonal Sannigrahi <sannigrahisonal@gmail.com>
Date: Sat, 22 Mar 2025 11:25:22 +0000
Subject: [PATCH 053/112] update for French [alm_bench]

---
 lmms_eval/tasks/alm_bench/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lmms_eval/tasks/alm_bench/utils.py b/lmms_eval/tasks/alm_bench/utils.py
index 9ebc00749..9e1f31cbb 100644
--- a/lmms_eval/tasks/alm_bench/utils.py
+++ b/lmms_eval/tasks/alm_bench/utils.py
@@ -27,7 +27,7 @@ def split_answer_options(text):
         "German": "Optionen",
     }
     text = text.strip()
-    match = re.match(r"^(.*?)\s*\((?:Options|Opties|옵션|选项|Opciones|opzioni|Варианты|choix|Opções|Optionen):\s*(.*?)\)$", text, re.IGNORECASE)
+    match = re.match(r"^(.*?)\s*\((?:Options|Opties|옵션|选项|Opciones|opzioni|Варианты|choix |Opções|Optionen):\s*(.*?)\)$", text, re.IGNORECASE)
     if match:
         true_answer = match.group(1).strip()
         choices = re.sub("\s*,\s*", "\n", match.group(2))
@@ -53,4 +53,4 @@ def alm_bench_process_results(doc, results):
 
 def alm_bench_doc_to_target(doc, model_specific_target_kwargs):
     true_answer, _ = split_answer_options(doc["Translated_Answer"])
-    return true_answer
\ No newline at end of file
+    return true_answer

From dbb1341d367d6d12fdc04f2b6601f8db82889f93 Mon Sep 17 00:00:00 2001
From: Sonal Sannigrahi <sannigrahisonal@gmail.com>
Date: Sat, 22 Mar 2025 11:26:45 +0000
Subject: [PATCH 054/112] Update alm_bench names

---
 lmms_eval/tasks/alm_bench/alm_bench-all.yaml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/lmms_eval/tasks/alm_bench/alm_bench-all.yaml b/lmms_eval/tasks/alm_bench/alm_bench-all.yaml
index d15f41fa2..b53eb0250 100644
--- a/lmms_eval/tasks/alm_bench/alm_bench-all.yaml
+++ b/lmms_eval/tasks/alm_bench/alm_bench-all.yaml
@@ -1,11 +1,11 @@
 group: alm_bench-all
 task:
-  - alm_bench-de
-  - alm_bench-es
-  - alm_bench-fr
-  - alm_bench-it
-  - alm_bench-ko
-  - alm_bench-nl
-  - alm_bench-pt
-  - alm_bench-ru
-  - alm_bench-en
\ No newline at end of file
+  - alm-bench-de
+  - alm-bench-es
+  - alm-bench-fr
+  - alm-bench-it
+  - alm-bench-ko
+  - alm-bench-nl
+  - alm-bench-pt
+  - alm-bench-ru
+  - alm-bench-en

From bbe5646773944a585c52a9f7d54caa66599e8911 Mon Sep 17 00:00:00 2001
From: Sonal Sannigrahi <sannigrahisonal@gmail.com>
Date: Sat, 22 Mar 2025 11:27:14 +0000
Subject: [PATCH 055/112] Update alm_bench names

---
 .../tasks/alm_bench/{alm_bench-de.yaml => alm-bench-de.yaml}    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename lmms_eval/tasks/alm_bench/{alm_bench-de.yaml => alm-bench-de.yaml} (96%)

diff --git a/lmms_eval/tasks/alm_bench/alm_bench-de.yaml b/lmms_eval/tasks/alm_bench/alm-bench-de.yaml
similarity index 96%
rename from lmms_eval/tasks/alm_bench/alm_bench-de.yaml
rename to lmms_eval/tasks/alm_bench/alm-bench-de.yaml
index 3348e6e36..02c198b9b 100644
--- a/lmms_eval/tasks/alm_bench/alm_bench-de.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-de.yaml
@@ -24,4 +24,4 @@ lmms_eval_specific_kwargs:
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
\ No newline at end of file
+    post_prompt: " Answer:"

From 3e8adfebd831181db2dd739f4ce72e756d7063ac Mon Sep 17 00:00:00 2001
From: Sonal Sannigrahi <sannigrahisonal@gmail.com>
Date: Sat, 22 Mar 2025 11:27:45 +0000
Subject: [PATCH 056/112] Update alm_bench names

---
 .../tasks/alm_bench/{alm_bench-en.yaml => alm-bench-en.yaml}    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename lmms_eval/tasks/alm_bench/{alm_bench-en.yaml => alm-bench-en.yaml} (96%)

diff --git a/lmms_eval/tasks/alm_bench/alm_bench-en.yaml b/lmms_eval/tasks/alm_bench/alm-bench-en.yaml
similarity index 96%
rename from lmms_eval/tasks/alm_bench/alm_bench-en.yaml
rename to lmms_eval/tasks/alm_bench/alm-bench-en.yaml
index e87c05b45..fcb767166 100644
--- a/lmms_eval/tasks/alm_bench/alm_bench-en.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-en.yaml
@@ -24,4 +24,4 @@ lmms_eval_specific_kwargs:
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
\ No newline at end of file
+    post_prompt: " Answer:"

From f608c86f56af9ecbc6413e76fc987e326248afae Mon Sep 17 00:00:00 2001
From: Sonal Sannigrahi <sannigrahisonal@gmail.com>
Date: Sat, 22 Mar 2025 11:28:06 +0000
Subject: [PATCH 057/112] Update alm_bench names

---
 .../tasks/alm_bench/{alm_bench-es.yaml => alm-bench-es.yaml}    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename lmms_eval/tasks/alm_bench/{alm_bench-es.yaml => alm-bench-es.yaml} (96%)

diff --git a/lmms_eval/tasks/alm_bench/alm_bench-es.yaml b/lmms_eval/tasks/alm_bench/alm-bench-es.yaml
similarity index 96%
rename from lmms_eval/tasks/alm_bench/alm_bench-es.yaml
rename to lmms_eval/tasks/alm_bench/alm-bench-es.yaml
index b3dd093a5..423730d79 100644
--- a/lmms_eval/tasks/alm_bench/alm_bench-es.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-es.yaml
@@ -24,4 +24,4 @@ lmms_eval_specific_kwargs:
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
\ No newline at end of file
+    post_prompt: " Answer:"

From 60180bb60c95ca6d81ff6e54b41772fb3b5ba02b Mon Sep 17 00:00:00 2001
From: Sonal Sannigrahi <sannigrahisonal@gmail.com>
Date: Sat, 22 Mar 2025 11:28:25 +0000
Subject: [PATCH 058/112] Update alm_bench names

---
 .../tasks/alm_bench/{alm_bench-fr.yaml => alm-bench-fr.yaml}    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename lmms_eval/tasks/alm_bench/{alm_bench-fr.yaml => alm-bench-fr.yaml} (96%)

diff --git a/lmms_eval/tasks/alm_bench/alm_bench-fr.yaml b/lmms_eval/tasks/alm_bench/alm-bench-fr.yaml
similarity index 96%
rename from lmms_eval/tasks/alm_bench/alm_bench-fr.yaml
rename to lmms_eval/tasks/alm_bench/alm-bench-fr.yaml
index 816e737d4..24947b8e9 100644
--- a/lmms_eval/tasks/alm_bench/alm_bench-fr.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-fr.yaml
@@ -24,4 +24,4 @@ lmms_eval_specific_kwargs:
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
\ No newline at end of file
+    post_prompt: " Answer:"

From 415fbe763ed5cb91827cfda3d9836cfbd4cf39c9 Mon Sep 17 00:00:00 2001
From: Sonal Sannigrahi <sannigrahisonal@gmail.com>
Date: Sat, 22 Mar 2025 11:28:44 +0000
Subject: [PATCH 059/112] Update alm_bench names

---
 .../tasks/alm_bench/{alm_bench-it.yaml => alm-bench-it.yaml}    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename lmms_eval/tasks/alm_bench/{alm_bench-it.yaml => alm-bench-it.yaml} (96%)

diff --git a/lmms_eval/tasks/alm_bench/alm_bench-it.yaml b/lmms_eval/tasks/alm_bench/alm-bench-it.yaml
similarity index 96%
rename from lmms_eval/tasks/alm_bench/alm_bench-it.yaml
rename to lmms_eval/tasks/alm_bench/alm-bench-it.yaml
index e48c1c3dc..956089c1d 100644
--- a/lmms_eval/tasks/alm_bench/alm_bench-it.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-it.yaml
@@ -24,4 +24,4 @@ lmms_eval_specific_kwargs:
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
\ No newline at end of file
+    post_prompt: " Answer:"

From 12620aa06792bcb1c672c9a5f54b9da286daec79 Mon Sep 17 00:00:00 2001
From: Sonal Sannigrahi <sannigrahisonal@gmail.com>
Date: Sat, 22 Mar 2025 11:29:03 +0000
Subject: [PATCH 060/112] Update alm_bench names

---
 .../tasks/alm_bench/{alm_bench-ko.yaml => alm-bench-ko.yaml}    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename lmms_eval/tasks/alm_bench/{alm_bench-ko.yaml => alm-bench-ko.yaml} (96%)

diff --git a/lmms_eval/tasks/alm_bench/alm_bench-ko.yaml b/lmms_eval/tasks/alm_bench/alm-bench-ko.yaml
similarity index 96%
rename from lmms_eval/tasks/alm_bench/alm_bench-ko.yaml
rename to lmms_eval/tasks/alm_bench/alm-bench-ko.yaml
index 9ebf5003c..ad13c1e62 100644
--- a/lmms_eval/tasks/alm_bench/alm_bench-ko.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-ko.yaml
@@ -24,4 +24,4 @@ lmms_eval_specific_kwargs:
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
\ No newline at end of file
+    post_prompt: " Answer:"

From bd98094615331ad98fa98d95110fad25a793d2ae Mon Sep 17 00:00:00 2001
From: Sonal Sannigrahi <sannigrahisonal@gmail.com>
Date: Sat, 22 Mar 2025 11:29:20 +0000
Subject: [PATCH 061/112] Update alm_bench names

---
 .../tasks/alm_bench/{alm_bench-nl.yaml => alm-bench-nl.yaml}    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename lmms_eval/tasks/alm_bench/{alm_bench-nl.yaml => alm-bench-nl.yaml} (96%)

diff --git a/lmms_eval/tasks/alm_bench/alm_bench-nl.yaml b/lmms_eval/tasks/alm_bench/alm-bench-nl.yaml
similarity index 96%
rename from lmms_eval/tasks/alm_bench/alm_bench-nl.yaml
rename to lmms_eval/tasks/alm_bench/alm-bench-nl.yaml
index cca54bad9..271d72b04 100644
--- a/lmms_eval/tasks/alm_bench/alm_bench-nl.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-nl.yaml
@@ -24,4 +24,4 @@ lmms_eval_specific_kwargs:
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
\ No newline at end of file
+    post_prompt: " Answer:"

From 534b45da852321e9f9a414e68242e55d62b425ff Mon Sep 17 00:00:00 2001
From: Sonal Sannigrahi <sannigrahisonal@gmail.com>
Date: Sat, 22 Mar 2025 11:29:40 +0000
Subject: [PATCH 062/112] Rename alm_bench-pt.yaml to alm-bench-pt.yaml

---
 .../tasks/alm_bench/{alm_bench-pt.yaml => alm-bench-pt.yaml}    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename lmms_eval/tasks/alm_bench/{alm_bench-pt.yaml => alm-bench-pt.yaml} (96%)

diff --git a/lmms_eval/tasks/alm_bench/alm_bench-pt.yaml b/lmms_eval/tasks/alm_bench/alm-bench-pt.yaml
similarity index 96%
rename from lmms_eval/tasks/alm_bench/alm_bench-pt.yaml
rename to lmms_eval/tasks/alm_bench/alm-bench-pt.yaml
index e942f3926..bf453f135 100644
--- a/lmms_eval/tasks/alm_bench/alm_bench-pt.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-pt.yaml
@@ -24,4 +24,4 @@ lmms_eval_specific_kwargs:
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
\ No newline at end of file
+    post_prompt: " Answer:"

From c001f3d7f9ecf15a42828400768a0a29d6a26de8 Mon Sep 17 00:00:00 2001
From: Sonal Sannigrahi <sannigrahisonal@gmail.com>
Date: Sat, 22 Mar 2025 11:30:12 +0000
Subject: [PATCH 063/112] Update alm_bench names

---
 .../tasks/alm_bench/{alm_bench-ru.yaml => alm-bench-ru.yaml}    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename lmms_eval/tasks/alm_bench/{alm_bench-ru.yaml => alm-bench-ru.yaml} (96%)

diff --git a/lmms_eval/tasks/alm_bench/alm_bench-ru.yaml b/lmms_eval/tasks/alm_bench/alm-bench-ru.yaml
similarity index 96%
rename from lmms_eval/tasks/alm_bench/alm_bench-ru.yaml
rename to lmms_eval/tasks/alm_bench/alm-bench-ru.yaml
index c26f2c19b..4f9ec6fa6 100644
--- a/lmms_eval/tasks/alm_bench/alm_bench-ru.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-ru.yaml
@@ -24,4 +24,4 @@ lmms_eval_specific_kwargs:
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
\ No newline at end of file
+    post_prompt: " Answer:"

From bc3032cdd8b0ef28b79246d4adcef329558c1b63 Mon Sep 17 00:00:00 2001
From: Sonal Sannigrahi <sannigrahisonal@gmail.com>
Date: Sat, 22 Mar 2025 11:31:04 +0000
Subject: [PATCH 064/112] Update alm_bench names

---
 scripts/parse_results.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/scripts/parse_results.py b/scripts/parse_results.py
index bec388a21..ead9ee282 100644
--- a/scripts/parse_results.py
+++ b/scripts/parse_results.py
@@ -264,15 +264,15 @@ def get_results_commute(results, task_name, model_name):
 def get_results_alm_bench(results, task_name, model_name):
     rows = []
     scores = {}
-    scores["de"] = float(results["alm_bench-de"]["exact_match,none"]) * 100
-    scores["es"] = float(results["alm_bench-es"]["exact_match,none"]) * 100
-    scores["fr"] = float(results["alm_bench-fr"]["exact_match,none"]) * 100
-    scores["it"] = float(results["alm_bench-it"]["exact_match,none"]) * 100
-    scores["ko"] = float(results["alm_bench-ko"]["exact_match,none"]) * 100
-    scores["nl"] = float(results["alm_bench-nl"]["exact_match,none"]) * 100
-    scores["pt"] = float(results["alm_bench-pt"]["exact_match,none"]) * 100
-    scores["ru"] = float(results["alm_bench-ru"]["exact_match,none"]) * 100
-    scores["en"] = float(results["alm_bench-en"]["exact_match,none"]) * 100
+    scores["de"] = float(results["alm-bench-de"]["exact_match,none"]) * 100
+    scores["es"] = float(results["alm-bench-es"]["exact_match,none"]) * 100
+    scores["fr"] = float(results["alm-bench-fr"]["exact_match,none"]) * 100
+    scores["it"] = float(results["alm-bench-it"]["exact_match,none"]) * 100
+    scores["ko"] = float(results["alm-bench-ko"]["exact_match,none"]) * 100
+    scores["nl"] = float(results["alm-bench-nl"]["exact_match,none"]) * 100
+    scores["pt"] = float(results["alm-bench-pt"]["exact_match,none"]) * 100
+    scores["ru"] = float(results["alm-bench-ru"]["exact_match,none"]) * 100
+    scores["en"] = float(results["alm-bench-en"]["exact_match,none"]) * 100
     
     for kk, vv in scores.items():
         rows.append({

From 9231ac6cc819a88af621b9374108318d36eb3560 Mon Sep 17 00:00:00 2001
From: farajian <amin@unbabel.com>
Date: Sat, 22 Mar 2025 18:50:36 +0000
Subject: [PATCH 065/112] updated gemini_api

---
 lmms_eval/models/gemini_api.py | 92 ++++++++++++++++------------------
 1 file changed, 43 insertions(+), 49 deletions(-)

diff --git a/lmms_eval/models/gemini_api.py b/lmms_eval/models/gemini_api.py
index 69f520c11..cdd1283de 100644
--- a/lmms_eval/models/gemini_api.py
+++ b/lmms_eval/models/gemini_api.py
@@ -9,22 +9,21 @@
 from loguru import logger as eval_logger
 from PIL import Image
 from tqdm import tqdm
-
+import io
+import base64
 from lmms_eval.api.instance import Instance
 from lmms_eval.api.model import lmms
 from lmms_eval.api.registry import register_model
 
-try:
-    import google.generativeai as genai
-    from google.generativeai.types import HarmBlockThreshold, HarmCategory
+from google import genai
+from google.genai import types
+# import google.generativeai as genai
+# from google.generativeai.types import HarmBlockThreshold, HarmCategory
 
-    NUM_SECONDS_TO_SLEEP = 30
-    GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
-    genai.configure(api_key=GOOGLE_API_KEY)
+NUM_SECONDS_TO_SLEEP = 30
+GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
+# genai.configure(api_key=GOOGLE_API_KEY)
 
-except Exception as e:
-    eval_logger.error(f"Error importing generativeai: {str(e)}")
-    genai = None
 
 try:
     import soundfile as sf
@@ -37,17 +36,18 @@ class GeminiAPI(lmms):
     def __init__(
         self,
         model_version: str = "gemini-1.5-pro",
-        # modality: str = "image",
+        modality: str = "image",
         timeout: int = 120,
         continual_mode: bool = False,
         response_persistent_folder: str = "./logs/gemini_persistent_folder",
-        # We will cache the Gemini API response in this path and use it for future requests
+        system_prompt: str = None,
         **kwargs,
     ) -> None:
         super().__init__()
         self.model_version = model_version
         self.timeout = timeout
-        self.model = genai.GenerativeModel(model_version)
+        # self.model = genai.GenerativeModel(model_version)
+        self.model = genai.Client(api_key=GOOGLE_API_KEY)
         self.continual_mode = continual_mode
         # if self.continual_mode and response_persistent_folder is None:
         #     raise ValueError("Continual mode requires a persistent path for the response. We will cache the Gemini API response in this path and use it for future requests. Please provide a valid path.")
@@ -57,13 +57,13 @@ def __init__(
                 os.makedirs(self.response_persistent_folder)
             self.response_persistent_file = os.path.join(self.response_persistent_folder, f"{self.model_version}_response.json")
 
-        if os.path.exists(self.response_persistent_file):
-            with open(self.response_persistent_file, "r") as f:
-                self.response_cache = json.load(f)
-            self.cache_mode = "resume"
-        else:
-            self.response_cache = {}
-            self.cache_mode = "start"
+        # if os.path.exists(self.response_persistent_file):
+        #     with open(self.response_persistent_file, "r") as f:
+        #         self.response_cache = json.load(f)
+        #     self.cache_mode = "resume"
+        # else:
+        #     self.response_cache = {}
+        self.cache_mode = "start"
 
         accelerator = Accelerator()
         if accelerator.num_processes > 1:
@@ -81,9 +81,10 @@ def __init__(
 
         self.device = self.accelerator.device
 
-        # self.modality = modality
+        self.modality = modality
 
         self.video_pool = []
+        self.system_prompt = system_prompt
 
     def free_video(self):
         for video in self.video_pool:
@@ -132,6 +133,14 @@ def convert_modality(self, images):
                     eval_logger.error(f"Error converting video: {str(e)}")
         return images
 
+    def get_image_url(self, image_bytes: bytes) -> str:
+        buffer = io.BytesIO()
+        image_bytes.save(buffer, format="JPEG")
+        img_bytes = buffer.getvalue()
+        base64_image = base64.b64encode(img_bytes).decode('utf-8')
+        data_url = f"data:image/jpeg;base64,{base64_image}"
+        return data_url
+
     def generate_until(self, requests) -> List[str]:
         res = []
         pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
@@ -140,51 +149,36 @@ def get_uuid(task, split, doc_id):
             return f"{task}___{split}___{doc_id}"
 
         for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
-            if self.continual_mode and self.cache_mode == "resume":
-                doc_uuid = get_uuid(task, split, doc_id)
-                if doc_uuid in self.response_cache:
-                    content = self.response_cache[doc_uuid]
-                    if content:
-                        res.append(content)
-                        pbar.update(1)
-                        continue
+
 
             if "max_new_tokens" not in gen_kwargs:
                 gen_kwargs["max_new_tokens"] = 1024
             if "temperature" not in gen_kwargs:
                 gen_kwargs["temperature"] = 0
 
-            config = genai.GenerationConfig(
-                max_output_tokens=gen_kwargs["max_new_tokens"],
-                temperature=gen_kwargs["temperature"],
-            )
-
             visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
-            visuals = self.flatten(visuals)
+
             visuals = self.convert_modality(visuals)
 
             message = [contexts] + visuals
 
             for attempt in range(5):
                 try:
-                    content = self.model.generate_content(
-                        message,
-                        generation_config=config,
-                        safety_settings={
-                            HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE,
-                            HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE,
-                            HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE,
-                            HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE,
-                        },
-                    )
-                    content = content.text
+                    response = self.model.models.generate_content(
+                        model=self.model_version,
+                        config=types.GenerateContentConfig(
+                        system_instruction=self.system_prompt,
+                        ),
+                        contents=message
+                        )
+                    results = response.text
                     break
                 except Exception as e:
                     eval_logger.info(f"Attempt {attempt + 1} failed with error: {str(e)}")
                     if isinstance(e, ValueError):
                         try:
-                            eval_logger.info(f"Prompt feed_back: {content.prompt_feedback}")
-                            content = ""
+                            eval_logger.info(f"Prompt feed_back: {results.prompt_feedback}")
+                            results = ""
                             break
                         except Exception:
                             pass
@@ -193,7 +187,7 @@ def get_uuid(task, split, doc_id):
                     else:  # If this was the last attempt, log and return empty
                         eval_logger.error(f"All 5 attempts failed. Last error message: {str(e)}")
                         content = ""
-            res.append(content)
+            res.append(results)
             pbar.update(1)
 
             self.free_video()

From 71afb48438370a597c0a921f677d588e19a5713d Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Thu, 27 Mar 2025 01:58:42 +0000
Subject: [PATCH 066/112] update to handle empty images

---
 lmms_eval/models/llava_hf.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lmms_eval/models/llava_hf.py b/lmms_eval/models/llava_hf.py
index 9c8bdfb79..9e9153aa2 100644
--- a/lmms_eval/models/llava_hf.py
+++ b/lmms_eval/models/llava_hf.py
@@ -319,6 +319,8 @@ def _collate(x):
                     image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visuals)
                 elif task_type == "video":
                     image_tokens = [DEFAULT_VIDEO_TOKEN] * len(visuals)
+                elif task_type == "text":
+                    image_tokens = []
                 image_tokens = " ".join(image_tokens)
                 context = f"{image_tokens}\n{context}"
             # Apply chat template
@@ -347,7 +349,8 @@ def _collate(x):
                 inputs = self._image_processor(images=visuals, text=text, return_tensors="pt").to(self._device, self.model.dtype)
             elif task_type == "video":
                 inputs = self._image_processor(videos=visuals, text=text, return_tensors="pt").to(self._device, self.model.dtype)
-
+            elif task_type == "text":
+                inputs = self._image_processor(text=text, return_tensors="pt").to(self._device, self.model.dtype)
             gen_kwargs["image_sizes"] = [visuals[idx].size for idx in range(len(visuals))]
             if "max_new_tokens" not in gen_kwargs:
                 gen_kwargs["max_new_tokens"] = 1024

From 9fd60b7fb08abc494e9b16f69914ff306a7eaf5b Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Thu, 27 Mar 2025 01:59:16 +0000
Subject: [PATCH 067/112] added wmt24pp

---
 lmms_eval/tasks/wmt24pp/comet_utils/comet.py  |  98 +++
 .../wmt24pp/comet_utils/metrics_handler.py    |  75 +++
 .../tasks/wmt24pp/comet_utils/myutils.py      | 611 ++++++++++++++++++
 .../wmt24pp/comet_utils/result_handler.py     |  39 ++
 lmms_eval/tasks/wmt24pp/utils.py              |  47 ++
 lmms_eval/tasks/wmt24pp/wmt24.cs-en.yaml      |  40 ++
 lmms_eval/tasks/wmt24pp/wmt24.de-en.yaml      |  41 ++
 lmms_eval/tasks/wmt24pp/wmt24.en-cs.yaml      |  40 ++
 lmms_eval/tasks/wmt24pp/wmt24.en-de.yaml      |  41 ++
 lmms_eval/tasks/wmt24pp/wmt24.en-es.yaml      |  40 ++
 lmms_eval/tasks/wmt24pp/wmt24.en-fr.yaml      |  41 ++
 lmms_eval/tasks/wmt24pp/wmt24.en-it.yaml      |  40 ++
 lmms_eval/tasks/wmt24pp/wmt24.en-ko.yaml      |  40 ++
 lmms_eval/tasks/wmt24pp/wmt24.en-nl.yaml      |  40 ++
 lmms_eval/tasks/wmt24pp/wmt24.en-pt.yaml      |  40 ++
 lmms_eval/tasks/wmt24pp/wmt24.en-ru.yaml      |  40 ++
 lmms_eval/tasks/wmt24pp/wmt24.en-zh.yaml      |  40 ++
 lmms_eval/tasks/wmt24pp/wmt24.es-en.yaml      |  40 ++
 lmms_eval/tasks/wmt24pp/wmt24.fr-en.yaml      |  41 ++
 lmms_eval/tasks/wmt24pp/wmt24.it-en.yaml      |  40 ++
 lmms_eval/tasks/wmt24pp/wmt24.ko-en.yaml      |  40 ++
 lmms_eval/tasks/wmt24pp/wmt24.nl-en.yaml      |  40 ++
 lmms_eval/tasks/wmt24pp/wmt24.pt-en.yaml      |  40 ++
 lmms_eval/tasks/wmt24pp/wmt24.ru-en.yaml      |  40 ++
 lmms_eval/tasks/wmt24pp/wmt24.zh-en.yaml      |  40 ++
 lmms_eval/tasks/wmt24pp/wmt24pp.yaml          |  23 +
 26 files changed, 1697 insertions(+)
 create mode 100644 lmms_eval/tasks/wmt24pp/comet_utils/comet.py
 create mode 100644 lmms_eval/tasks/wmt24pp/comet_utils/metrics_handler.py
 create mode 100644 lmms_eval/tasks/wmt24pp/comet_utils/myutils.py
 create mode 100644 lmms_eval/tasks/wmt24pp/comet_utils/result_handler.py
 create mode 100644 lmms_eval/tasks/wmt24pp/utils.py
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24.cs-en.yaml
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24.de-en.yaml
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24.en-cs.yaml
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24.en-de.yaml
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24.en-es.yaml
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24.en-fr.yaml
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24.en-it.yaml
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24.en-ko.yaml
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24.en-nl.yaml
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24.en-pt.yaml
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24.en-ru.yaml
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24.en-zh.yaml
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24.es-en.yaml
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24.fr-en.yaml
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24.it-en.yaml
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24.ko-en.yaml
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24.nl-en.yaml
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24.pt-en.yaml
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24.ru-en.yaml
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24.zh-en.yaml
 create mode 100644 lmms_eval/tasks/wmt24pp/wmt24pp.yaml

diff --git a/lmms_eval/tasks/wmt24pp/comet_utils/comet.py b/lmms_eval/tasks/wmt24pp/comet_utils/comet.py
new file mode 100644
index 000000000..f88f4aee0
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/comet_utils/comet.py
@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+from comet import download_model, load_from_checkpoint
+
+from lmms_eval.tasks.multi30k.comet_utils.metrics_handler import Metric
+from lmms_eval.tasks.multi30k.comet_utils.result_handler import MetricResult
+
+
+class BaseCOMETResult(MetricResult):
+    """
+    COMET Result Handler.
+    """
+
+    def __init__(
+        self,
+        result: float,
+    ) -> None:
+        super().__init__(result)
+
+
+class BaseCOMET(Metric):
+    def __init__(self, model: str, **kwargs) -> None:
+        super().__init__(**kwargs)
+        model_path = download_model(model)
+        self.model = load_from_checkpoint(model_path)
+        self.model.eval()
+
+    def load_gold_data(self, gold_data):
+        pass
+
+    def make_samples(
+        self, sources: list[str], hypotheses: list[str], references: list[str] = None
+    ):
+        pass
+
+    def run(
+        self,
+        hypothesis_path,
+        gold_data_path,
+        gpus: int = 1,
+        batch_size: int = 16,
+        **kwargs
+    ) -> dict:
+        hypotheses, gold_data = self._handle_inputs(hypothesis_path, gold_data_path)
+        references, sources = self.load_gold_data(gold_data)
+        result = self.evaluate(hypotheses, references, sources, gpus, batch_size)
+        result.print_result(self.metric_name())
+        return result.format_result(self.metric_name())
+
+    def evaluate(
+        self, hypotheses: list, references: list, sources: list, gpus, batch_size
+    ) -> BaseCOMETResult:
+        """
+        Evaluate function receives the hypotheses and the references and returns a COMETResult object.
+
+        :param hypotheses: List of the MT outputs (sentences).
+        :param references: List of the reference sentences.
+        :param sources: List of source sentences
+        """
+        samples = self.make_samples(sources, hypotheses, references)
+
+        outputs = self.model.predict(
+            samples=samples,
+            batch_size=batch_size,
+            gpus=gpus,
+            accelerator="auto",
+        )
+        system_score, segments_scores = outputs.system_score, outputs.scores
+
+        comet_result = BaseCOMETResult(
+            {
+                "system_score": system_score,
+                "segments_scores": segments_scores,
+            }
+        )
+        return comet_result
+
+    def process_result(self, result) -> MetricResult:
+        pass
+
+    @staticmethod
+    def metric_name():
+        pass
+
+
+class RefCOMET(BaseCOMET):
+    def __init__(self, model: str, **kwargs) -> None:
+        super().__init__(model, **kwargs)
+
+    def load_gold_data(self, gold_data):
+        references, sources = gold_data["ref"], gold_data["src"]
+        return references, sources
+
+    def make_samples(
+        self, sources: list[str], hypotheses: list[str], references: list[str]
+    ):
+        samples = {"src": sources, "mt": hypotheses, "ref": references}
+        samples = [dict(zip(samples, t)) for t in zip(*samples.values())]
+        return samples
\ No newline at end of file
diff --git a/lmms_eval/tasks/wmt24pp/comet_utils/metrics_handler.py b/lmms_eval/tasks/wmt24pp/comet_utils/metrics_handler.py
new file mode 100644
index 000000000..c8ab1560e
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/comet_utils/metrics_handler.py
@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+from lmms_eval.tasks.multi30k.comet_utils.result_handler import MetricResult
+from lmms_eval.tasks.multi30k.comet_utils.myutils import list_to_dict, load_jsonl_file, read_lines
+
+
+class Metric(ABC):
+    """Abstract class defining a shared interface for all the Metrics"""
+
+    def __init__(self, **kwargs) -> None:
+        self.kwargs = kwargs
+
+    def run(self, hypothesis_path: Path = None, references_path: Path = None, **kwargs):
+        """
+        The runner function that performs all the necessary checks on the files,
+        applies the needed preprocessing (like lowercasing, tokenization, etc),
+        and calls the evaluate method of the class to get the scores.
+        """
+        pass
+
+    @abstractmethod
+    def evaluate(self, hypotheses: list, references: list, **kwargs) -> MetricResult:
+        """
+        Evaluate function receives the hypotheses and the reference files and returns a MetricResult object.
+
+        :param hypotheses: the path to the hypothese file.
+        :param references: the path to the reference file.
+        :return: MetricResult object.
+        """
+        pass
+
+    @abstractmethod
+    def process_result(self, result) -> MetricResult:
+        """
+        Process the result to be ready and complient with the MetricResult format.
+
+        :param result: the raw result produced by the metric
+        :return: MetricResult object.
+        """
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def metric_name() -> None:
+        """Metric name used to address the metric via cli."""
+        pass
+
+    @staticmethod
+    def _handle_inputs(
+        hypotheses: Path,
+        data_path: Path,
+    ) -> tuple:
+        """
+        Function to handle input files.
+        All inputs will be returned as list of strings.
+
+        :param hypotheses: either the handler to the file storing the hypotheses.
+        :param references: either the handler to the file storing the refereneces.
+        :param sources: either the handler to the file storing the source sentences.
+
+        :return:
+            -  Tuple with hypotheses, references and kwargs
+            -  If sources not None, Tuple with hypotheses, references, sources and kwargs
+        """
+        hypotheses = read_lines(hypotheses, unescape_newline=True)
+        # gold data keys depend on the task; e.g., for MT, it will include "ref", for APE "pe"
+        gold_data = load_jsonl_file(data_path)
+        assert len(hypotheses) == len(
+            gold_data
+        ), f"The number of hypotheses ({len(hypotheses)}) and rows in the gold data ({len(gold_data)}) should be the same."
+        gold_data = list_to_dict(gold_data)
+
+        return hypotheses, gold_data
\ No newline at end of file
diff --git a/lmms_eval/tasks/wmt24pp/comet_utils/myutils.py b/lmms_eval/tasks/wmt24pp/comet_utils/myutils.py
new file mode 100644
index 000000000..4adf7eaef
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/comet_utils/myutils.py
@@ -0,0 +1,611 @@
+import argparse
+import ast
+import json
+import os
+import re
+import subprocess
+import time
+from pathlib import Path
+from typing import Dict, Iterable, List, Union
+
+import lxml
+import numpy as np
+import spacy
+import tqdm
+import yaml
+from loguru import logger
+from lxml import etree
+# from mosestokenizer import MosesDetokenizer
+from nltk.tokenize.treebank import TreebankWordDetokenizer
+from tenacity import Retrying, stop_after_attempt, wait_random_exponential
+
+PathInput = Union[str, Path]
+PATTERN_OPEN_TAG = r"<\s*([\w]+)\s*>\s*"
+PATTERN_CLOSE_TAG = r"\s*<\s*\/\s*([\w]+)\s*>"
+PATTERN_SHOT_NAME = r"^([0-9]+)[-_]shot.*$"
+
+SPACY_LANGUAGE_TO_MODEL = {
+    "en": "en_core_web_sm",
+    "de": "de_core_news_sm",
+    "fr": "fr_core_news_sm",
+    "nl": "nl_core_news_sm",
+    "pt": "pt_core_news_sm",
+    "ru": "ru_core_news_sm",
+    "it": "it_core_news_sm",
+    "pl": "pl_core_news_sm",
+    "es": "es_core_news_sm",
+    "sv": "sv_core_news_sm",
+    "zh": "zh_core_web_sm",
+    "ko": "ko_core_news_sm",
+}
+
+
+def args_to_dict(args, prefix: str, strip_prefix: bool = False):
+    """Filters argparse's `Namespace` into dictionary with arguments
+    beginning with the given prefix."""
+    prefix += "_"
+    d = {}
+    for k, v in args.__dict__.items():
+        if k.startswith(prefix) or k in ["language", "tokenize", "lowercase"]:
+            k = k.replace(prefix, "") if strip_prefix else k
+            d[k] = v
+    return d
+
+
+def detokenize(lines: list[str]) -> list[str]:
+    detokenizer = TreebankWordDetokenizer()
+    detokenized_lines = [detokenizer.detokenize(line.split()) for line in lines]
+    return detokenized_lines
+
+
+def detokenize_moses(lines: list[str], lang: str) -> list[str]:
+    detokenizer = MosesDetokenizer(lang)
+    detokenized_lines = [detokenizer(line.split()) for line in lines]
+    return detokenized_lines
+
+
+def tokenize_spacy(lines: list[str], language, keep_xml=False) -> list[str]:
+    spacy_model = SPACY_LANGUAGE_TO_MODEL[language]
+    try:
+        tokenizer = spacy.load(spacy_model, disable=["tagger", "parser", "ner"])
+    except OSError:
+        logger.warning(f"Spacy model {spacy_model} not found. Trying to download...")
+        download_proc = subprocess.run(
+            ["python", "-m", "spacy", "download", spacy_model]
+        )
+        tokenizer = spacy.load(spacy_model, disable=["tagger", "parser", "ner"])
+
+    tokenized = list(tokenizer.pipe(lines))
+    tokenized = [[token.text for token in sentence] for sentence in tokenized]
+    output_lines = [" ".join(sentence) for sentence in tokenized]
+    if keep_xml:
+        output_lines = [
+            re.sub(PATTERN_OPEN_TAG, r"<\1>", output_line)
+            for output_line in output_lines
+        ]
+        output_lines = [
+            re.sub(PATTERN_CLOSE_TAG, r"</\1>", output_line)
+            for output_line in output_lines
+        ]
+    return output_lines
+
+
+def parse_yaml_config(config_path):
+    with open(config_path, "r") as config_file:
+        config_data = yaml.safe_load(config_file)
+    return config_data
+
+
+def get_task_args(args: dict) -> dict:
+    source = args.get("source")
+    hypothesis = args.get("hypothesis")
+    references = args.get("references")
+
+    task_args = {
+        "hypothesis": hypothesis,
+        "references": references,
+    }
+    if source:
+        task_args.update({"source": source})
+    return task_args
+
+
+def load_json_file(
+    file_path: PathInput,
+) -> list:
+    """
+    Function to to load json files.
+
+    :param file_path: either the handler to the file storing the hypotheses.
+    """
+    with open(file_path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def load_jsonl_file(
+    file_path: PathInput,
+) -> list:
+    """
+    Function to parse error span jsonl files.
+    All inputs will be returned as list of dictionaries.
+
+    :param file_path: either the handler to the file storing the hypotheses.
+        Must be a jsonl file where each line is a dict which has at least the following keys:
+        {"mt": <mt>, "annotations": [{"start": <start>, "end": <end>, "severity": <severity>}, ...]}
+    :return:
+        -  List of dictionaries
+    """
+    with open(file_path, "r", encoding="utf-8") as f:
+        json_list = list(f)
+
+    return [json.loads(l) for l in json_list]
+
+
+def list_to_dict(l: list[dict[str, str]]) -> dict[str, list[str]]:
+    """
+    Assumes lists in dict have the same length.
+    """
+    d = {}
+    for k in l[0].keys():
+        d[k] = [d[k] for d in l]
+    return d
+
+
+def dict_to_records(d: dict[str, list[str]]) -> list[dict[str, str]]:
+    """
+    Assumes lists in dict have the same length.
+    """
+    records = []
+    for i in range(len(list(d.values())[0])):
+        records.append({k: v[i] for k, v in d.items()})
+
+    return records
+
+
+def write_lines(
+    path: PathInput,
+    lines: Iterable[str],
+    escape_newline: bool = False,
+    escape_return_char: bool = True,
+    verbose: bool = True,
+) -> None:
+    """Writes lines to a file.
+
+    Lines can be escaped, meaning \n is transformed to \\n.
+
+    Args:
+        path: The path to the file.
+        lines: The lines to write.
+        escape_newline: Whether to escape newlines.
+    """
+    # make dir, if not exists
+    if isinstance(path, str):
+        path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    out_lines = []
+    for i, line in enumerate(lines):
+        if escape_return_char:
+            if "\r" in line and verbose:
+                logger.opt(ansi=True).warning(
+                    f"Detected carriage return in line {i + 1} (\\r). This may cause errors downstream. Escaping. This behaviour is the default; you can turn it off with escape_return_char."
+                )
+            line = line.replace("\r", "\\r")
+        if escape_newline:
+            if "\n" in line and verbose:
+                logger.opt(ansi=True).warning(
+                    f"Found new line in line {i + 1} (\\n). This may cause errors downstream. Escaping."
+                )
+            line = line.replace("\n", "\\n")
+        out_lines.append(line)
+    with open(path, "w", encoding="utf-8") as f:
+        f.writelines((f"{l}\n" for l in out_lines))
+
+
+def combine_metrics_args(task_metric: dict, subtask_metric: dict) -> dict:
+    eval_metric = task_metric.copy()
+    for metric_arg, metric_arg_value in subtask_metric.items():
+        eval_metric[metric_arg] = metric_arg_value
+    return eval_metric
+
+
+def generate_with_retries(
+    retry_function,
+    model_args,
+    retry_max_attempts=1,
+    retry_multiplier=1,
+    retry_max_interval=10,
+    retry_min_interval=4,
+):
+    retryer = Retrying(
+        stop=stop_after_attempt(retry_max_attempts),
+        wait=wait_random_exponential(
+            multiplier=retry_multiplier, max=retry_max_interval, min=retry_min_interval
+        ),
+        reraise=True,
+    )
+    return retryer(retry_function, **model_args)
+
+
+def sample_strings_from_list(strings: list[str], num_samples: int) -> list[str]:
+    """ """
+    return np.random.choice(strings, num_samples).tolist()
+
+
+def read_lines(path: PathInput, unescape_newline: bool = False) -> List[str]:
+    """Reads lines from a file.
+    Lines can be unescapped, meaning \\n is transformed to \n.
+    Args:
+        path: The path to the file.
+        unescape_newline: Whether to unescape newlines.
+    Returns:
+        The lines in the file."""
+    with open(path, encoding="utf-8") as f:
+        lines = [l[:-1] for l in f.readlines()]
+    if unescape_newline:
+        lines = [l.replace("\\n", "\n") for l in lines]
+    return lines
+
+
+def make_dir_if_not_exists(output_file):
+    output_dir = os.path.dirname(output_file)
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+
+def log_response(response: str, step: int, lim: int = 10) -> None:
+    """
+    Logs the response from the model.
+    """
+    if step < lim:
+        if "<" in response:
+            response = response.replace("<", "\\<")
+            logger.opt(colors=True).info(
+                f"<red> Escaped tags in below response to not throw colorizing error. </red>"
+            )
+        logger.opt(colors=True).info(f"Response {step}: {response.strip()}")
+
+
+def get_num_processed_lines(output_file: str, **kwargs):
+    """
+    This function is used to write the output from the last line of the input file.
+    """
+    if os.path.exists(output_file):
+        processed_lines = read_lines(output_file, unescape_newline=False)
+        num_processed_lines = len(processed_lines)
+        if num_processed_lines > 0:
+            logger.opt(ansi=True).warning(
+                f"<red>Resuming generation from line {num_processed_lines}.</red>"
+            )
+    else:
+        num_processed_lines = 0
+
+    return num_processed_lines
+
+
+def load_data_to_records(
+    data_path: str,
+) -> list[dict[str, str]]:
+    """ """
+    # prompt data file can either be a jsonl or a csv
+    data = load_jsonl_file(data_path)
+
+    return data
+
+
+def save_to_json(
+    save_location: PathInput,
+    data: dict,
+) -> None:
+    """ """
+    if isinstance(save_location, str):
+        save_location = Path(save_location)
+    save_location.parent.mkdir(parents=True, exist_ok=True)
+    with open(save_location, "w") as output_path:
+        json.dump(data, output_path, indent=4)
+
+
+def get_sacrebleu_segment_scores(
+    hypotheses: list[str],
+    references,
+    method,
+) -> list[float]:
+    """ """
+    segment_scores = []
+    for h, r in zip(hypotheses, references):
+        segment_score = method.sentence_score(h, r)
+        segment_scores.append(segment_score.score)
+    return segment_scores
+
+
+def get_eval_args_given_task(
+    eval_args: Dict[str, str],
+    task_name: str,
+    subtask: str,
+    gen_output_dir: Path,
+    eval_data_dir: Path,
+    eval_output_dir: Path,
+    model_type: str,
+    model_name: str,
+) -> Dict[str, str]:
+    """ """
+    if task_name == "gec":
+        eval_args["references_m2"] = eval_data_dir / task_name / subtask / "test_corpus.m2"
+    gen_output_path = (
+        gen_output_dir
+        / task_name
+        / subtask
+        / model_type
+        / model_name
+        / "generation.txt"
+    )
+
+    eval_output_path = (
+        eval_output_dir
+        / task_name
+        / subtask
+        / model_type
+        / model_name
+        / "evaluation.json"
+    )
+
+    eval_data_path = eval_data_dir / task_name / subtask / "test.jsonl"
+    # add language argument to eval_args as it is needed in some metrics
+    lp = subtask.split(".")[1]
+    src_lang, trg_lang = get_langs(lp)
+    eval_args["lp"] = {"src_lang": src_lang, "trg_lang": trg_lang}
+
+    return gen_output_path, eval_data_path, eval_output_path, eval_args
+
+
+def text_to_label(
+    text: str,
+    text_type: str,
+    label_space: List[str] = None,
+    return_is_random: bool = False,
+) -> int:
+    is_random = False
+    if text_type == "categorical":
+        label = int(text.strip())
+        is_random = False
+    # if hypothesis is text
+    elif text_type == "text":
+        label = np.random.randint(len(label_space))
+        is_random = True
+        for l in label_space:
+            if re.search(l, text, re.IGNORECASE) is not None:
+                label = label_space.index(l)
+                is_random = False
+                break
+    if return_is_random:
+        return label, is_random
+    else:
+        return label
+
+
+def handle_subprocess(subprocess_args: List[str], check_output: bool = False):
+    output = 1
+    if check_output:
+        process = subprocess.Popen(
+            subprocess_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
+        )
+        output = ""
+
+        # Read and print the output in real-time
+        for line in iter(process.stdout.readline, ""):
+            print(line, end="")  # Print each line as it's received
+            output += line  # Append each line to the output variable
+    else:
+        try:
+            process = subprocess.Popen(subprocess_args)
+            while process.poll() is None:
+                time.sleep(1)
+            logger.info("Generation process has finished.")
+            output = 0
+        except KeyboardInterrupt:
+            process.terminate()
+    return output
+
+
+def tokenize_text(references, tokenizer):
+    tokenized_prompts = []
+    logger.info(f"Tokenizing text...")
+    for reference in tqdm.tqdm(references):
+        tokenized_prompts.append(tokenizer.encode(reference))
+    return tokenized_prompts
+
+
+def get_langs(lp):
+    # TODO: This has to be updated to account for more languages
+    valid_langs = [
+        "de",
+        "en-us",
+        "en-gb",
+        "en-uk",
+        "en",
+        "es-latam",
+        "es",
+        "fr",
+        "fr-ca",
+        "fr-fr",
+        "it",
+        "it-it",
+        "ko",
+        "ko-ko",
+        "nl",
+        "nl-nl",
+        "pl",
+        "pl-pl",
+        "pt-br",
+        "pt",
+        "pt-pt",
+        "ru",
+        "zh-CN",
+        "zh-cn",
+        "zh-TW",
+        "zh-tw",
+        "zh",
+        "ja",
+        "hi",
+        "uk",
+        "cs",
+        "is",
+        "sv",
+        "sv-se",
+        "ar",
+        "tr",
+        "el",
+        "hu",
+        "hu-hu",
+        "ro",
+        "ro-ro",
+        "fi",
+        "fi-fi",
+        "sk",
+        "da",
+        "da-da",
+        "lt",
+        "lv",
+        "sl",
+        "et",
+        "bg",
+        "no",
+        "no-no",
+        "ca",
+        "hr",
+        "ga",
+        "mt",
+        "gl",
+    ]
+    lang_pattern = "|".join(valid_langs)
+    lp_pattern = rf"^({lang_pattern})-({lang_pattern})$"
+    match = re.match(lp_pattern, lp)
+    if match:
+        # We have a language pair, so both src_lang and trg_lang will be extracted from lp
+        src_lang = match.group(1)
+        trg_lang = match.group(2)
+        return src_lang, trg_lang
+    elif lp in valid_langs:
+        # the task is monolingual, hence we only have one language. So, we set the src_lang only. trg_lang will be set to None
+        src_lang = lp
+        trg_lang = None
+        return src_lang, trg_lang
+    return None, None
+
+
+def add_average_generation_time(
+    input_file: str, metadata_file: str, language: str, mode: str = "lps"
+):
+    """
+    Given the generation file, the metadata file and the mode it calculates the time to generate the outputs.
+    Depending on the mode, it can be either the lines-per-second (lps) or words-per-second (wps)
+
+    Args:
+        input_file (str): The generation file
+        metadata_file (str): The metadata file that contains the generation_time which is a list
+        mode (str, optional): The mode for calculating the time. It can be either the lines-per-second (lps) or words-per-second (wps). Defaults to "lps".
+    """
+    lines = read_lines(input_file)
+    metadata = load_json_file(metadata_file)
+    generation_time = metadata.get("generation_time")
+    if generation_time:
+        total_time = sum(generation_time)
+        if mode == "lps":
+            average_time = len(lines) / total_time
+        # TODO: Add the support of wps mode
+    else:
+        logger.error(
+            f"The generation_time information doesn't exist in the {metadata_file}. We are going to set the average generation time to -1."
+        )
+        average_time = -1
+    metadata[f"generation_time_average ({mode})"] = average_time
+    save_to_json(metadata_file, metadata)
+
+
+def parse_dict_arg(arg):
+    try:
+        # Safely evaluate the string as a Python expression
+        parsed = ast.literal_eval(arg)
+        if isinstance(parsed, dict):
+            return parsed
+        else:
+            raise argparse.ArgumentTypeError("Argument is not a valid dictionary")
+    except (ValueError, SyntaxError):
+        raise argparse.ArgumentTypeError("Argument is not a valid dictionary")
+
+
+def write_jsonl(path, data):
+    with open(path, "w") as f:
+        for d in data:
+            f.write(json.dumps(d, ensure_ascii=False) + "\n")
+
+
+def match_xml(string_1: lxml.etree._Element, string_2: lxml.etree._Element):
+    if string_1.tag != string_2.tag:
+        return False
+
+    string_1 = list(string_1.iterchildren())
+    string_2 = list(string_2.iterchildren())
+
+    if len(string_1) != len(string_2):
+        return False
+
+    for s1, s2 in zip(string_1, string_2):
+        if not match_xml(s1, s2):
+            return False
+    return True
+
+
+def split_by_xml_tags(string: str):
+    try:
+        root = etree.fromstring(f"<root>{string}</root>")
+        parts = []
+
+        for element in root.iter():
+            if element.tag == "root":
+                if element.text:
+                    parts.append(element.text)
+                continue
+
+            if element.text:
+                parts.append(element.text)
+            if element.tail:
+                parts.append(element.tail)
+
+        return [p.strip() for p in parts if p.strip()]
+    except etree.XMLSyntaxError:
+        return [string]
+
+
+def split_pairs_by_xml_tags(string1: str, string2: str):
+    try:
+        string1_xml = etree.fromstring(f"<root>{string1}</root>")
+        string2_xml = etree.fromstring(f"<root>{string2}</root>")
+        if not match_xml(string1_xml, string2_xml):
+            return (None, None)
+
+        string1_segmented = split_by_xml_tags(string1)
+        string2_segmented = split_by_xml_tags(string2)
+        string1_segmented = [seg for seg in string1_segmented if seg is not None]
+        string2_segmented = [seg for seg in string2_segmented if seg is not None]
+
+        if len(string1_segmented) == len(string2_segmented):
+            return (string1_segmented, string2_segmented)
+    except:
+        pass
+
+    return (None, None)
+
+def prepare_xml_markup_pairs(hypotheses, references):
+    hypothesis_segmented, references_segmented = [], []
+    non_matching_indices = []
+    current_index = 0
+    for hyp, ref in zip(hypotheses, references):
+        hs, rs = split_pairs_by_xml_tags(hyp, ref)
+        if hs and rs:
+            hypothesis_segmented.extend(hs)
+            references_segmented.extend(rs)
+            current_index += len(hs)
+        else:
+            non_matching_indices.append(current_index)
+            current_index += 1
+    return hypothesis_segmented, references_segmented, non_matching_indices
\ No newline at end of file
diff --git a/lmms_eval/tasks/wmt24pp/comet_utils/result_handler.py b/lmms_eval/tasks/wmt24pp/comet_utils/result_handler.py
new file mode 100644
index 000000000..9b571d909
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/comet_utils/result_handler.py
@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+from abc import ABC
+
+
+class MetricResult(ABC):
+    """
+    Abstract class defining a shared interface for all Metric Result Handlers.
+
+    :param result: float value to be displayed.
+    """
+
+    def __init__(self, result: float) -> None:
+        self.result = result
+
+    def print_result(self, metric_name: str, round_to_decimals: int = 4) -> None:
+        """Function used to display a particular Metric result.
+        :param metric_name: Metric name.
+        :param round_to_decimals: decimals that we want to present.
+        """
+        if type(self.result) == dict:
+            print(
+                f'{metric_name}: {round(self.result["system_score"], round_to_decimals)}'
+            )
+        else:
+            print(f"{metric_name}: {round(self.result, round_to_decimals)}")
+
+    def format_result(self, metric_name: str, round_to_decimals: int = 4) -> dict:
+        """Function used to format a particular Metric result.
+        :param metric_name: Metric name.
+        :param round_to_decimals: decimals that we want to present.
+        """
+        if type(self.result) == dict:
+            out = {
+                f"{metric_name}": round(self.result["system_score"], round_to_decimals),
+                f"{metric_name}_segments": self.result["segments_scores"],
+            }
+        else:
+            out = {f"{metric_name}": round(self.result, round_to_decimals)}
+        return out
\ No newline at end of file
diff --git a/lmms_eval/tasks/wmt24pp/utils.py b/lmms_eval/tasks/wmt24pp/utils.py
new file mode 100644
index 000000000..1fb19f127
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/utils.py
@@ -0,0 +1,47 @@
+import numpy as np
+from loguru import logger as eval_logger
+from lmms_eval.tasks.multi30k.comet_utils.comet import RefCOMET
+
+def process_docs(docs):
+    # docs = docs.select(range(10))
+    docs = docs.filter(lambda x: x["is_bad_source"] != "true")
+    return docs
+
+def process_docs_reverse(docs):
+    docs = docs.select(range(10))
+    # docs = docs.filter(lambda x: x["is_bad_source"] != "true")
+    # Swap source and target
+    docs = docs.map(lambda x: {
+        **x,  # Keep all other fields
+        'source': x['target'],
+        'target': x['source']
+    })
+    return docs
+
+def doc_to_text(doc, lmms_eval_specific_kwargs):
+    source_txt = doc["source"]
+    pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    pre_prompt = pre_prompt.format(source=source_txt)
+    return f"{pre_prompt}"
+
+def doc_to_visual(doc):
+    return []
+
+def process_results(doc, results):
+    pred_answer = results[0]
+    source = doc["source"]
+    target = doc["target"]
+    return {"results":{"prediction":pred_answer,"ground_truth": target, "source":source}}
+
+def aggregate_results(results):
+    comet = RefCOMET(model="Unbabel/XCOMET-XL")
+    sources = [res["source"] for res in results]
+    hypotheses = [res["prediction"] for res in results]
+    references = [res["ground_truth"] for res in results]
+    comet.make_samples(sources, hypotheses, references)
+    segments_scores_correct = comet.evaluate(hypotheses, references, sources, gpus=1, batch_size=16).result["segments_scores"]
+    results = {
+               f"avg_XCOMET-XL_score": np.mean(segments_scores_correct),
+               }
+    return results
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.cs-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.cs-en.yaml
new file mode 100644
index 000000000..e354fa009
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24.cs-en.yaml
@@ -0,0 +1,40 @@
+dataset_path: google/wmt24pp
+task: "wmt24pp.cs-en"
+test_split: "train"
+dataset_name: "en-cs_CZ"
+
+output_type: generate_until
+# process_docs: !function utils.process_docs
+doc_to_visual: !function utils.doc_to_visual
+doc_to_text: !function utils.doc_to_text_reverse
+doc_to_target: "en"
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.process_results_reverse
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from Czech to English:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.de-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.de-en.yaml
new file mode 100644
index 000000000..34c6c15a0
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24.de-en.yaml
@@ -0,0 +1,41 @@
+dataset_path: google/wmt24pp
+task: "wmt24pp.de-en"
+test_split: "train"
+dataset_name: "en-de_DE"
+
+output_type: generate_until
+# process_docs: !function utils.process_docs
+doc_to_visual: !function utils.doc_to_visual
+doc_to_text: !function utils.doc_to_text_reverse
+doc_to_target: "en"
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.process_results_reverse
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    eval_model: "Unbabel/XCOMET-XXL"
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from German to English:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.en-cs.yaml b/lmms_eval/tasks/wmt24pp/wmt24.en-cs.yaml
new file mode 100644
index 000000000..00e3d6a4e
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24.en-cs.yaml
@@ -0,0 +1,40 @@
+dataset_path: google/wmt24pp
+task: "wmt24pp.en-cs"
+test_split: "train"
+dataset_name: "en-cs_CZ"
+
+output_type: generate_until
+# process_docs: !function utils.process_docs
+doc_to_visual: !function utils.doc_to_visual
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "cs"
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.process_results
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from English to Czech:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.en-de.yaml b/lmms_eval/tasks/wmt24pp/wmt24.en-de.yaml
new file mode 100644
index 000000000..3568661e6
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24.en-de.yaml
@@ -0,0 +1,41 @@
+dataset_path: google/wmt24pp
+task: "wmt24pp.en-de"
+test_split: "train"
+dataset_name: "en-de_DE"
+
+output_type: generate_until
+# process_docs: !function utils.process_docs
+doc_to_visual: !function utils.doc_to_visual
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "de"
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.process_results
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    eval_model: "Unbabel/XCOMET-XXL"
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from English to German:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.en-es.yaml b/lmms_eval/tasks/wmt24pp/wmt24.en-es.yaml
new file mode 100644
index 000000000..068c54bbe
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24.en-es.yaml
@@ -0,0 +1,40 @@
+dataset_path: google/wmt24pp
+task: "wmt24pp.en-es"
+test_split: "train"
+dataset_name: "en-es_MX"
+
+output_type: generate_until
+# process_docs: !function utils.process_docs
+doc_to_visual: !function utils.doc_to_visual
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "es"
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.process_results
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from English to Spanish:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.en-fr.yaml b/lmms_eval/tasks/wmt24pp/wmt24.en-fr.yaml
new file mode 100644
index 000000000..873e9c121
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24.en-fr.yaml
@@ -0,0 +1,41 @@
+dataset_path: google/wmt24pp
+task: "wmt24pp.en-fr"
+test_split: "train"
+dataset_name: "en-fr_FR"
+
+
+output_type: generate_until
+# process_docs: !function utils.process_docs
+doc_to_visual: !function utils.doc_to_visual
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "fr"
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.process_results
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from English to French:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.en-it.yaml b/lmms_eval/tasks/wmt24pp/wmt24.en-it.yaml
new file mode 100644
index 000000000..d8aae90e5
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24.en-it.yaml
@@ -0,0 +1,40 @@
+dataset_path: google/wmt24pp
+task: "wmt24pp.en-it"
+test_split: "train"
+dataset_name: "en-it_IT"
+
+output_type: generate_until
+# process_docs: !function utils.process_docs
+doc_to_visual: !function utils.doc_to_visual
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "it"
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.process_results
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from English to Italian:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.en-ko.yaml b/lmms_eval/tasks/wmt24pp/wmt24.en-ko.yaml
new file mode 100644
index 000000000..eb5258a58
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24.en-ko.yaml
@@ -0,0 +1,40 @@
+dataset_path: google/wmt24pp
+task: "wmt24pp.en-ko"
+test_split: "train"
+dataset_name: "en-ko_KR"
+
+output_type: generate_until
+# process_docs: !function utils.process_docs
+doc_to_visual: !function utils.doc_to_visual
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "ko"
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.process_results
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from English to Korean:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.en-nl.yaml b/lmms_eval/tasks/wmt24pp/wmt24.en-nl.yaml
new file mode 100644
index 000000000..ffb921aac
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24.en-nl.yaml
@@ -0,0 +1,40 @@
+dataset_path: google/wmt24pp
+task: "wmt24pp.en-nl"
+test_split: "train"
+dataset_name: "en-nl_NL"
+
+output_type: generate_until
+# process_docs: !function utils.process_docs
+doc_to_visual: !function utils.doc_to_visual
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "nl"
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.process_results
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from English to Dutch:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.en-pt.yaml b/lmms_eval/tasks/wmt24pp/wmt24.en-pt.yaml
new file mode 100644
index 000000000..2d3067e05
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24.en-pt.yaml
@@ -0,0 +1,40 @@
+dataset_path: google/wmt24pp
+task: "wmt24pp.en-pt"
+test_split: "train"
+dataset_name: "en-pt_PT"
+
+output_type: generate_until
+# process_docs: !function utils.process_docs
+doc_to_visual: !function utils.doc_to_visual
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "pt"
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.process_results
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from English to Portuguese:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.en-ru.yaml b/lmms_eval/tasks/wmt24pp/wmt24.en-ru.yaml
new file mode 100644
index 000000000..da6148151
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24.en-ru.yaml
@@ -0,0 +1,40 @@
+dataset_path: google/wmt24pp
+task: "wmt24pp.en-ru"
+test_split: "train"
+dataset_name: "en-ru_RU"
+
+output_type: generate_until
+# process_docs: !function utils.process_docs
+doc_to_visual: !function utils.doc_to_visual
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "ru"
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.process_results
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from English to Russian:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.en-zh.yaml b/lmms_eval/tasks/wmt24pp/wmt24.en-zh.yaml
new file mode 100644
index 000000000..19c385f0d
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24.en-zh.yaml
@@ -0,0 +1,40 @@
+dataset_path: google/wmt24pp
+task: "wmt24pp.en-zh"
+test_split: "train"
+dataset_name: "en-zh_CN"
+
+output_type: generate_until
+# process_docs: !function utils.process_docs
+doc_to_visual: !function utils.doc_to_visual
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "zh"
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.process_results
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from English to Chinese:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.es-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.es-en.yaml
new file mode 100644
index 000000000..7e4d85761
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24.es-en.yaml
@@ -0,0 +1,40 @@
+dataset_path: google/wmt24pp
+task: "wmt24pp.es-en"
+test_split: "train"
+dataset_name: "en-es_MX"
+
+output_type: generate_until
+process_docs: !function utils.process_docs_reverse
+doc_to_visual: !function utils.doc_to_visual
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "en"
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.process_results
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from Spanish to English:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.fr-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.fr-en.yaml
new file mode 100644
index 000000000..65f9dd0ac
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24.fr-en.yaml
@@ -0,0 +1,41 @@
+dataset_path: google/wmt24pp
+task: "wmt24pp.fr-en"
+test_split: "train"
+dataset_name: "en-fr_FR"
+
+
+output_type: generate_until
+process_docs: !function utils.process_docs_reverse
+doc_to_visual: !function utils.doc_to_visual
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "en"
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.process_results
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from French to English:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.it-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.it-en.yaml
new file mode 100644
index 000000000..9c68dd2ae
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24.it-en.yaml
@@ -0,0 +1,40 @@
+dataset_path: google/wmt24pp
+task: "wmt24pp.it-en"
+test_split: "train"
+dataset_name: "en-it_IT"
+
+output_type: generate_until
+process_docs: !function utils.process_docs_reverse
+doc_to_visual: !function utils.doc_to_visual
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "en"
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.process_results
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from Italian to English:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.ko-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.ko-en.yaml
new file mode 100644
index 000000000..9f684423b
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24.ko-en.yaml
@@ -0,0 +1,40 @@
+dataset_path: google/wmt24pp
+task: "wmt24pp.ko-en"
+test_split: "train"
+dataset_name: "en-ko_KR"
+
+output_type: generate_until
+process_docs: !function utils.process_docs_reverse
+doc_to_visual: !function utils.doc_to_visual
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "en"
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.process_results
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:  
+  default:
+    pre_prompt: "Translate the following source text from Korean to English:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.nl-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.nl-en.yaml
new file mode 100644
index 000000000..45e687cde
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24.nl-en.yaml
@@ -0,0 +1,40 @@
+dataset_path: google/wmt24pp
+task: "wmt24pp.nl-en"
+test_split: "train"
+dataset_name: "en-nl_NL"
+
+output_type: generate_until
+process_docs: !function utils.process_docs_reverse
+doc_to_visual: !function utils.doc_to_visual
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "en"
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.process_results
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from Dutch to English:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.pt-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.pt-en.yaml
new file mode 100644
index 000000000..820903747
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24.pt-en.yaml
@@ -0,0 +1,40 @@
+dataset_path: google/wmt24pp
+task: "wmt24pp.pt-en"
+test_split: "train"
+dataset_name: "en-pt_PT"
+
+output_type: generate_until
+process_docs: !function utils.process_docs_reverse
+doc_to_visual: !function utils.doc_to_visual
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "en"
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.process_results
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from Portuguese to English:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.ru-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.ru-en.yaml
new file mode 100644
index 000000000..647072755
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24.ru-en.yaml
@@ -0,0 +1,40 @@
+dataset_path: google/wmt24pp
+task: "wmt24pp.ru-en"
+test_split: "train"
+dataset_name: "en-ru_RU"
+
+output_type: generate_until
+process_docs: !function utils.process_docs_reverse
+doc_to_visual: !function utils.doc_to_visual
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "en"
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.process_results
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from Russian to English:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.zh-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.zh-en.yaml
new file mode 100644
index 000000000..94c7d09d5
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24.zh-en.yaml
@@ -0,0 +1,40 @@
+dataset_path: google/wmt24pp
+task: "wmt24pp.zh-en"
+test_split: "train"
+dataset_name: "en-zh_CN"
+
+output_type: generate_until
+process_docs: !function utils.process_docs_reverse
+doc_to_visual: !function utils.doc_to_visual
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "en"
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.process_results
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Translate the following source text from Chinese to English:\n{source}"
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+
+
+
diff --git a/lmms_eval/tasks/wmt24pp/wmt24pp.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.yaml
new file mode 100644
index 000000000..5c536c77b
--- /dev/null
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.yaml
@@ -0,0 +1,23 @@
+group: wmt24pp
+task:
+  # - wmt24pp.en-de
+  # - wmt24pp.en-fr
+  # - wmt24pp.en-cs
+  # - wmt24pp.en-pt
+  # - wmt24pp.en-it
+  # - wmt24pp.en-ko
+  # - wmt24pp.en-nl
+  # - wmt24pp.en-ru
+  # - wmt24pp.en-es
+  # - wmt24pp.en-zh
+  # - wmt24pp.de-en
+  # - wmt24pp.fr-en
+  # - wmt24pp.cs-en
+  # - wmt24pp.pt-en
+  # - wmt24pp.it-en
+  # - wmt24pp.ko-en
+  # - wmt24pp.nl-en
+  # - wmt24pp.ru-en
+  # - wmt24pp.es-en
+  - wmt24pp.zh-en
+

From 79c2215bfa62d29ce651f5732b9683720d00fd2c Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Thu, 27 Mar 2025 02:05:34 +0000
Subject: [PATCH 068/112] small update on wmt24pp

---
 lmms_eval/tasks/wmt24pp/utils.py     |  4 +--
 lmms_eval/tasks/wmt24pp/wmt24pp.yaml | 38 ++++++++++++++--------------
 2 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/lmms_eval/tasks/wmt24pp/utils.py b/lmms_eval/tasks/wmt24pp/utils.py
index 1fb19f127..2a3584e0d 100644
--- a/lmms_eval/tasks/wmt24pp/utils.py
+++ b/lmms_eval/tasks/wmt24pp/utils.py
@@ -4,11 +4,11 @@
 
 def process_docs(docs):
     # docs = docs.select(range(10))
-    docs = docs.filter(lambda x: x["is_bad_source"] != "true")
+    # docs = docs.filter(lambda x: x["is_bad_source"] != "true")
     return docs
 
 def process_docs_reverse(docs):
-    docs = docs.select(range(10))
+    # docs = docs.select(range(10))
     # docs = docs.filter(lambda x: x["is_bad_source"] != "true")
     # Swap source and target
     docs = docs.map(lambda x: {
diff --git a/lmms_eval/tasks/wmt24pp/wmt24pp.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.yaml
index 5c536c77b..3b268340b 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24pp.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.yaml
@@ -1,23 +1,23 @@
 group: wmt24pp
 task:
-  # - wmt24pp.en-de
-  # - wmt24pp.en-fr
-  # - wmt24pp.en-cs
-  # - wmt24pp.en-pt
-  # - wmt24pp.en-it
-  # - wmt24pp.en-ko
-  # - wmt24pp.en-nl
-  # - wmt24pp.en-ru
-  # - wmt24pp.en-es
-  # - wmt24pp.en-zh
-  # - wmt24pp.de-en
-  # - wmt24pp.fr-en
-  # - wmt24pp.cs-en
-  # - wmt24pp.pt-en
-  # - wmt24pp.it-en
-  # - wmt24pp.ko-en
-  # - wmt24pp.nl-en
-  # - wmt24pp.ru-en
-  # - wmt24pp.es-en
+  - wmt24pp.en-de
+  - wmt24pp.en-fr
+  - wmt24pp.en-cs
+  - wmt24pp.en-pt
+  - wmt24pp.en-it
+  - wmt24pp.en-ko
+  - wmt24pp.en-nl
+  - wmt24pp.en-ru
+  - wmt24pp.en-es
+  - wmt24pp.en-zh
+  - wmt24pp.de-en
+  - wmt24pp.fr-en
+  - wmt24pp.cs-en
+  - wmt24pp.pt-en
+  - wmt24pp.it-en
+  - wmt24pp.ko-en
+  - wmt24pp.nl-en
+  - wmt24pp.ru-en
+  - wmt24pp.es-en
   - wmt24pp.zh-en
 

From 7ff1712b2bb82c0929a224ad2daf90a270435b1b Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Thu, 27 Mar 2025 02:13:14 +0000
Subject: [PATCH 069/112] fixed yaml wmt24pp

---
 lmms_eval/tasks/wmt24pp/wmt24.cs-en.yaml | 6 +++---
 lmms_eval/tasks/wmt24pp/wmt24.de-en.yaml | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/lmms_eval/tasks/wmt24pp/wmt24.cs-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.cs-en.yaml
index e354fa009..d8b46fde8 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24.cs-en.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24.cs-en.yaml
@@ -4,9 +4,9 @@ test_split: "train"
 dataset_name: "en-cs_CZ"
 
 output_type: generate_until
-# process_docs: !function utils.process_docs
+process_docs: !function utils.process_docs_reverse
 doc_to_visual: !function utils.doc_to_visual
-doc_to_text: !function utils.doc_to_text_reverse
+doc_to_text: !function utils.doc_to_text
 doc_to_target: "en"
 
 generation_kwargs:
@@ -16,7 +16,7 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-process_results: !function utils.process_results_reverse
+process_results: !function utils.process_results
 
 metric_list:
   - metric: results
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.de-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.de-en.yaml
index 34c6c15a0..5f41e57b5 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24.de-en.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24.de-en.yaml
@@ -4,9 +4,9 @@ test_split: "train"
 dataset_name: "en-de_DE"
 
 output_type: generate_until
-# process_docs: !function utils.process_docs
+process_docs: !function utils.process_docs_reverse
 doc_to_visual: !function utils.doc_to_visual
-doc_to_text: !function utils.doc_to_text_reverse
+doc_to_text: !function utils.doc_to_text
 doc_to_target: "en"
 
 generation_kwargs:
@@ -16,7 +16,7 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-process_results: !function utils.process_results_reverse
+process_results: !function utils.process_results
 
 metric_list:
   - metric: results

From 1b6b6670a3b64cb65298d9bfeec21d2fe47a250e Mon Sep 17 00:00:00 2001
From: amin <amin@unbabel.com>
Date: Fri, 28 Mar 2025 14:56:15 +0000
Subject: [PATCH 070/112] updated wmt24pp group

---
 lmms_eval/tasks/wmt24pp/utils.py              | 15 +------
 lmms_eval/tasks/wmt24pp/wmt24.cs-en.yaml      | 40 ------------------
 lmms_eval/tasks/wmt24pp/wmt24.de-en.yaml      | 41 -------------------
 lmms_eval/tasks/wmt24pp/wmt24.es-en.yaml      | 40 ------------------
 lmms_eval/tasks/wmt24pp/wmt24.fr-en.yaml      | 41 -------------------
 lmms_eval/tasks/wmt24pp/wmt24.it-en.yaml      | 40 ------------------
 lmms_eval/tasks/wmt24pp/wmt24.ko-en.yaml      | 40 ------------------
 lmms_eval/tasks/wmt24pp/wmt24.nl-en.yaml      | 40 ------------------
 lmms_eval/tasks/wmt24pp/wmt24.pt-en.yaml      | 40 ------------------
 lmms_eval/tasks/wmt24pp/wmt24.ru-en.yaml      | 40 ------------------
 lmms_eval/tasks/wmt24pp/wmt24.zh-en.yaml      | 40 ------------------
 .../{wmt24.en-cs.yaml => wmt24pp.en-cs.yaml}  | 10 +----
 .../{wmt24.en-de.yaml => wmt24pp.en-de.yaml}  | 10 +----
 .../{wmt24.en-es.yaml => wmt24pp.en-es.yaml}  | 10 +----
 .../{wmt24.en-fr.yaml => wmt24pp.en-fr.yaml}  | 10 +----
 .../{wmt24.en-it.yaml => wmt24pp.en-it.yaml}  | 10 +----
 .../{wmt24.en-ko.yaml => wmt24pp.en-ko.yaml}  | 10 +----
 .../{wmt24.en-nl.yaml => wmt24pp.en-nl.yaml}  | 10 +----
 .../{wmt24.en-pt.yaml => wmt24pp.en-pt.yaml}  | 10 +----
 .../{wmt24.en-ru.yaml => wmt24pp.en-ru.yaml}  | 10 +----
 .../{wmt24.en-zh.yaml => wmt24pp.en-zh.yaml}  | 10 +----
 lmms_eval/tasks/wmt24pp/wmt24pp.yaml          | 16 ++------
 22 files changed, 25 insertions(+), 508 deletions(-)
 delete mode 100644 lmms_eval/tasks/wmt24pp/wmt24.cs-en.yaml
 delete mode 100644 lmms_eval/tasks/wmt24pp/wmt24.de-en.yaml
 delete mode 100644 lmms_eval/tasks/wmt24pp/wmt24.es-en.yaml
 delete mode 100644 lmms_eval/tasks/wmt24pp/wmt24.fr-en.yaml
 delete mode 100644 lmms_eval/tasks/wmt24pp/wmt24.it-en.yaml
 delete mode 100644 lmms_eval/tasks/wmt24pp/wmt24.ko-en.yaml
 delete mode 100644 lmms_eval/tasks/wmt24pp/wmt24.nl-en.yaml
 delete mode 100644 lmms_eval/tasks/wmt24pp/wmt24.pt-en.yaml
 delete mode 100644 lmms_eval/tasks/wmt24pp/wmt24.ru-en.yaml
 delete mode 100644 lmms_eval/tasks/wmt24pp/wmt24.zh-en.yaml
 rename lmms_eval/tasks/wmt24pp/{wmt24.en-cs.yaml => wmt24pp.en-cs.yaml} (79%)
 rename lmms_eval/tasks/wmt24pp/{wmt24.en-de.yaml => wmt24pp.en-de.yaml} (80%)
 rename lmms_eval/tasks/wmt24pp/{wmt24.en-es.yaml => wmt24pp.en-es.yaml} (79%)
 rename lmms_eval/tasks/wmt24pp/{wmt24.en-fr.yaml => wmt24pp.en-fr.yaml} (79%)
 rename lmms_eval/tasks/wmt24pp/{wmt24.en-it.yaml => wmt24pp.en-it.yaml} (79%)
 rename lmms_eval/tasks/wmt24pp/{wmt24.en-ko.yaml => wmt24pp.en-ko.yaml} (79%)
 rename lmms_eval/tasks/wmt24pp/{wmt24.en-nl.yaml => wmt24pp.en-nl.yaml} (79%)
 rename lmms_eval/tasks/wmt24pp/{wmt24.en-pt.yaml => wmt24pp.en-pt.yaml} (79%)
 rename lmms_eval/tasks/wmt24pp/{wmt24.en-ru.yaml => wmt24pp.en-ru.yaml} (79%)
 rename lmms_eval/tasks/wmt24pp/{wmt24.en-zh.yaml => wmt24pp.en-zh.yaml} (79%)

diff --git a/lmms_eval/tasks/wmt24pp/utils.py b/lmms_eval/tasks/wmt24pp/utils.py
index 2a3584e0d..bcb258671 100644
--- a/lmms_eval/tasks/wmt24pp/utils.py
+++ b/lmms_eval/tasks/wmt24pp/utils.py
@@ -1,5 +1,4 @@
 import numpy as np
-from loguru import logger as eval_logger
 from lmms_eval.tasks.multi30k.comet_utils.comet import RefCOMET
 
 def process_docs(docs):
@@ -7,21 +6,11 @@ def process_docs(docs):
     # docs = docs.filter(lambda x: x["is_bad_source"] != "true")
     return docs
 
-def process_docs_reverse(docs):
-    # docs = docs.select(range(10))
-    # docs = docs.filter(lambda x: x["is_bad_source"] != "true")
-    # Swap source and target
-    docs = docs.map(lambda x: {
-        **x,  # Keep all other fields
-        'source': x['target'],
-        'target': x['source']
-    })
-    return docs
-
 def doc_to_text(doc, lmms_eval_specific_kwargs):
     source_txt = doc["source"]
     pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
-    pre_prompt = pre_prompt.format(source=source_txt)
+    post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+    pre_prompt = f"{pre_prompt} {source_txt}\n{post_prompt.strip()} "
     return f"{pre_prompt}"
 
 def doc_to_visual(doc):
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.cs-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.cs-en.yaml
deleted file mode 100644
index d8b46fde8..000000000
--- a/lmms_eval/tasks/wmt24pp/wmt24.cs-en.yaml
+++ /dev/null
@@ -1,40 +0,0 @@
-dataset_path: google/wmt24pp
-task: "wmt24pp.cs-en"
-test_split: "train"
-dataset_name: "en-cs_CZ"
-
-output_type: generate_until
-process_docs: !function utils.process_docs_reverse
-doc_to_visual: !function utils.doc_to_visual
-doc_to_text: !function utils.doc_to_text
-doc_to_target: "en"
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-process_results: !function utils.process_results
-
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
-    higher_is_better: true
-    ignore_case: true
-    ignore_punctuation: true
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: "Translate the following source text from Czech to English:\n{source}"
-    post_prompt: ""
-  qwen_vl:
-    pre_prompt: ""
-    post_prompt: " Answer:"
-
-
-
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.de-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.de-en.yaml
deleted file mode 100644
index 5f41e57b5..000000000
--- a/lmms_eval/tasks/wmt24pp/wmt24.de-en.yaml
+++ /dev/null
@@ -1,41 +0,0 @@
-dataset_path: google/wmt24pp
-task: "wmt24pp.de-en"
-test_split: "train"
-dataset_name: "en-de_DE"
-
-output_type: generate_until
-process_docs: !function utils.process_docs_reverse
-doc_to_visual: !function utils.doc_to_visual
-doc_to_text: !function utils.doc_to_text
-doc_to_target: "en"
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-process_results: !function utils.process_results
-
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
-    higher_is_better: true
-    ignore_case: true
-    ignore_punctuation: true
-    eval_model: "Unbabel/XCOMET-XXL"
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: "Translate the following source text from German to English:\n{source}"
-    post_prompt: ""
-  qwen_vl:
-    pre_prompt: ""
-    post_prompt: " Answer:"
-
-
-
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.es-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.es-en.yaml
deleted file mode 100644
index 7e4d85761..000000000
--- a/lmms_eval/tasks/wmt24pp/wmt24.es-en.yaml
+++ /dev/null
@@ -1,40 +0,0 @@
-dataset_path: google/wmt24pp
-task: "wmt24pp.es-en"
-test_split: "train"
-dataset_name: "en-es_MX"
-
-output_type: generate_until
-process_docs: !function utils.process_docs_reverse
-doc_to_visual: !function utils.doc_to_visual
-doc_to_text: !function utils.doc_to_text
-doc_to_target: "en"
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-process_results: !function utils.process_results
-
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
-    higher_is_better: true
-    ignore_case: true
-    ignore_punctuation: true
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: "Translate the following source text from Spanish to English:\n{source}"
-    post_prompt: ""
-  qwen_vl:
-    pre_prompt: ""
-    post_prompt: " Answer:"
-
-
-
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.fr-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.fr-en.yaml
deleted file mode 100644
index 65f9dd0ac..000000000
--- a/lmms_eval/tasks/wmt24pp/wmt24.fr-en.yaml
+++ /dev/null
@@ -1,41 +0,0 @@
-dataset_path: google/wmt24pp
-task: "wmt24pp.fr-en"
-test_split: "train"
-dataset_name: "en-fr_FR"
-
-
-output_type: generate_until
-process_docs: !function utils.process_docs_reverse
-doc_to_visual: !function utils.doc_to_visual
-doc_to_text: !function utils.doc_to_text
-doc_to_target: "en"
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-process_results: !function utils.process_results
-
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
-    higher_is_better: true
-    ignore_case: true
-    ignore_punctuation: true
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: "Translate the following source text from French to English:\n{source}"
-    post_prompt: ""
-  qwen_vl:
-    pre_prompt: ""
-    post_prompt: " Answer:"
-
-
-
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.it-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.it-en.yaml
deleted file mode 100644
index 9c68dd2ae..000000000
--- a/lmms_eval/tasks/wmt24pp/wmt24.it-en.yaml
+++ /dev/null
@@ -1,40 +0,0 @@
-dataset_path: google/wmt24pp
-task: "wmt24pp.it-en"
-test_split: "train"
-dataset_name: "en-it_IT"
-
-output_type: generate_until
-process_docs: !function utils.process_docs_reverse
-doc_to_visual: !function utils.doc_to_visual
-doc_to_text: !function utils.doc_to_text
-doc_to_target: "en"
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-process_results: !function utils.process_results
-
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
-    higher_is_better: true
-    ignore_case: true
-    ignore_punctuation: true
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: "Translate the following source text from Italian to English:\n{source}"
-    post_prompt: ""
-  qwen_vl:
-    pre_prompt: ""
-    post_prompt: " Answer:"
-
-
-
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.ko-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.ko-en.yaml
deleted file mode 100644
index 9f684423b..000000000
--- a/lmms_eval/tasks/wmt24pp/wmt24.ko-en.yaml
+++ /dev/null
@@ -1,40 +0,0 @@
-dataset_path: google/wmt24pp
-task: "wmt24pp.ko-en"
-test_split: "train"
-dataset_name: "en-ko_KR"
-
-output_type: generate_until
-process_docs: !function utils.process_docs_reverse
-doc_to_visual: !function utils.doc_to_visual
-doc_to_text: !function utils.doc_to_text
-doc_to_target: "en"
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-process_results: !function utils.process_results
-
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
-    higher_is_better: true
-    ignore_case: true
-    ignore_punctuation: true
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:  
-  default:
-    pre_prompt: "Translate the following source text from Korean to English:\n{source}"
-    post_prompt: ""
-  qwen_vl:
-    pre_prompt: ""
-    post_prompt: " Answer:"
-
-
-
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.nl-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.nl-en.yaml
deleted file mode 100644
index 45e687cde..000000000
--- a/lmms_eval/tasks/wmt24pp/wmt24.nl-en.yaml
+++ /dev/null
@@ -1,40 +0,0 @@
-dataset_path: google/wmt24pp
-task: "wmt24pp.nl-en"
-test_split: "train"
-dataset_name: "en-nl_NL"
-
-output_type: generate_until
-process_docs: !function utils.process_docs_reverse
-doc_to_visual: !function utils.doc_to_visual
-doc_to_text: !function utils.doc_to_text
-doc_to_target: "en"
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-process_results: !function utils.process_results
-
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
-    higher_is_better: true
-    ignore_case: true
-    ignore_punctuation: true
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: "Translate the following source text from Dutch to English:\n{source}"
-    post_prompt: ""
-  qwen_vl:
-    pre_prompt: ""
-    post_prompt: " Answer:"
-
-
-
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.pt-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.pt-en.yaml
deleted file mode 100644
index 820903747..000000000
--- a/lmms_eval/tasks/wmt24pp/wmt24.pt-en.yaml
+++ /dev/null
@@ -1,40 +0,0 @@
-dataset_path: google/wmt24pp
-task: "wmt24pp.pt-en"
-test_split: "train"
-dataset_name: "en-pt_PT"
-
-output_type: generate_until
-process_docs: !function utils.process_docs_reverse
-doc_to_visual: !function utils.doc_to_visual
-doc_to_text: !function utils.doc_to_text
-doc_to_target: "en"
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-process_results: !function utils.process_results
-
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
-    higher_is_better: true
-    ignore_case: true
-    ignore_punctuation: true
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: "Translate the following source text from Portuguese to English:\n{source}"
-    post_prompt: ""
-  qwen_vl:
-    pre_prompt: ""
-    post_prompt: " Answer:"
-
-
-
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.ru-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.ru-en.yaml
deleted file mode 100644
index 647072755..000000000
--- a/lmms_eval/tasks/wmt24pp/wmt24.ru-en.yaml
+++ /dev/null
@@ -1,40 +0,0 @@
-dataset_path: google/wmt24pp
-task: "wmt24pp.ru-en"
-test_split: "train"
-dataset_name: "en-ru_RU"
-
-output_type: generate_until
-process_docs: !function utils.process_docs_reverse
-doc_to_visual: !function utils.doc_to_visual
-doc_to_text: !function utils.doc_to_text
-doc_to_target: "en"
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-process_results: !function utils.process_results
-
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
-    higher_is_better: true
-    ignore_case: true
-    ignore_punctuation: true
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: "Translate the following source text from Russian to English:\n{source}"
-    post_prompt: ""
-  qwen_vl:
-    pre_prompt: ""
-    post_prompt: " Answer:"
-
-
-
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.zh-en.yaml b/lmms_eval/tasks/wmt24pp/wmt24.zh-en.yaml
deleted file mode 100644
index 94c7d09d5..000000000
--- a/lmms_eval/tasks/wmt24pp/wmt24.zh-en.yaml
+++ /dev/null
@@ -1,40 +0,0 @@
-dataset_path: google/wmt24pp
-task: "wmt24pp.zh-en"
-test_split: "train"
-dataset_name: "en-zh_CN"
-
-output_type: generate_until
-process_docs: !function utils.process_docs_reverse
-doc_to_visual: !function utils.doc_to_visual
-doc_to_text: !function utils.doc_to_text
-doc_to_target: "en"
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-process_results: !function utils.process_results
-
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
-    higher_is_better: true
-    ignore_case: true
-    ignore_punctuation: true
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: "Translate the following source text from Chinese to English:\n{source}"
-    post_prompt: ""
-  qwen_vl:
-    pre_prompt: ""
-    post_prompt: " Answer:"
-
-
-
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.en-cs.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.en-cs.yaml
similarity index 79%
rename from lmms_eval/tasks/wmt24pp/wmt24.en-cs.yaml
rename to lmms_eval/tasks/wmt24pp/wmt24pp.en-cs.yaml
index 00e3d6a4e..a3ded92cf 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24.en-cs.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.en-cs.yaml
@@ -30,11 +30,5 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following source text from English to Czech:\n{source}"
-    post_prompt: ""
-  qwen_vl:
-    pre_prompt: ""
-    post_prompt: " Answer:"
-
-
-
+    pre_prompt: "Translate the following English source text to Czech.\nEnglish:"
+    post_prompt: "Czech:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.en-de.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.en-de.yaml
similarity index 80%
rename from lmms_eval/tasks/wmt24pp/wmt24.en-de.yaml
rename to lmms_eval/tasks/wmt24pp/wmt24pp.en-de.yaml
index 3568661e6..6cf76a0ad 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24.en-de.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.en-de.yaml
@@ -31,11 +31,5 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following source text from English to German:\n{source}"
-    post_prompt: ""
-  qwen_vl:
-    pre_prompt: ""
-    post_prompt: " Answer:"
-
-
-
+    pre_prompt: "Translate the following English source text to German.\nEnglish:"
+    post_prompt: "German:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.en-es.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.en-es.yaml
similarity index 79%
rename from lmms_eval/tasks/wmt24pp/wmt24.en-es.yaml
rename to lmms_eval/tasks/wmt24pp/wmt24pp.en-es.yaml
index 068c54bbe..73b4b9e91 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24.en-es.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.en-es.yaml
@@ -30,11 +30,5 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following source text from English to Spanish:\n{source}"
-    post_prompt: ""
-  qwen_vl:
-    pre_prompt: ""
-    post_prompt: " Answer:"
-
-
-
+    pre_prompt: "Translate the following English source text to Spanish.\nEnglish:"
+    post_prompt: "Spanish:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.en-fr.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.en-fr.yaml
similarity index 79%
rename from lmms_eval/tasks/wmt24pp/wmt24.en-fr.yaml
rename to lmms_eval/tasks/wmt24pp/wmt24pp.en-fr.yaml
index 873e9c121..f9c8d31a7 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24.en-fr.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.en-fr.yaml
@@ -31,11 +31,5 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following source text from English to French:\n{source}"
-    post_prompt: ""
-  qwen_vl:
-    pre_prompt: ""
-    post_prompt: " Answer:"
-
-
-
+    pre_prompt: "Translate the following English source text to French.\nEnglish:"
+    post_prompt: "French:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.en-it.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.en-it.yaml
similarity index 79%
rename from lmms_eval/tasks/wmt24pp/wmt24.en-it.yaml
rename to lmms_eval/tasks/wmt24pp/wmt24pp.en-it.yaml
index d8aae90e5..a2b541c2c 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24.en-it.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.en-it.yaml
@@ -30,11 +30,5 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following source text from English to Italian:\n{source}"
-    post_prompt: ""
-  qwen_vl:
-    pre_prompt: ""
-    post_prompt: " Answer:"
-
-
-
+    pre_prompt: "Translate the following English source text to Italian.\nEnglish:"
+    post_prompt: "Italian:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.en-ko.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.en-ko.yaml
similarity index 79%
rename from lmms_eval/tasks/wmt24pp/wmt24.en-ko.yaml
rename to lmms_eval/tasks/wmt24pp/wmt24pp.en-ko.yaml
index eb5258a58..3860025dc 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24.en-ko.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.en-ko.yaml
@@ -30,11 +30,5 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following source text from English to Korean:\n{source}"
-    post_prompt: ""
-  qwen_vl:
-    pre_prompt: ""
-    post_prompt: " Answer:"
-
-
-
+    pre_prompt: "Translate the following English source text to Korean.\nEnglish:"
+    post_prompt: "Korean:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.en-nl.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.en-nl.yaml
similarity index 79%
rename from lmms_eval/tasks/wmt24pp/wmt24.en-nl.yaml
rename to lmms_eval/tasks/wmt24pp/wmt24pp.en-nl.yaml
index ffb921aac..f758b3662 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24.en-nl.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.en-nl.yaml
@@ -30,11 +30,5 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following source text from English to Dutch:\n{source}"
-    post_prompt: ""
-  qwen_vl:
-    pre_prompt: ""
-    post_prompt: " Answer:"
-
-
-
+    pre_prompt: "Translate the following English source text to Dutch.\nEnglish:"
+    post_prompt: "Dutch:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.en-pt.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.en-pt.yaml
similarity index 79%
rename from lmms_eval/tasks/wmt24pp/wmt24.en-pt.yaml
rename to lmms_eval/tasks/wmt24pp/wmt24pp.en-pt.yaml
index 2d3067e05..9d705c7c2 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24.en-pt.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.en-pt.yaml
@@ -30,11 +30,5 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following source text from English to Portuguese:\n{source}"
-    post_prompt: ""
-  qwen_vl:
-    pre_prompt: ""
-    post_prompt: " Answer:"
-
-
-
+    pre_prompt: "Translate the following English source text to Portuguese.\nEnglish:"
+    post_prompt: "Portuguese:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.en-ru.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.en-ru.yaml
similarity index 79%
rename from lmms_eval/tasks/wmt24pp/wmt24.en-ru.yaml
rename to lmms_eval/tasks/wmt24pp/wmt24pp.en-ru.yaml
index da6148151..af25b583b 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24.en-ru.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.en-ru.yaml
@@ -30,11 +30,5 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following source text from English to Russian:\n{source}"
-    post_prompt: ""
-  qwen_vl:
-    pre_prompt: ""
-    post_prompt: " Answer:"
-
-
-
+    pre_prompt: "Translate the following English source text to Russian.\nEnglish:"
+    post_prompt: "Russian:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/wmt24pp/wmt24.en-zh.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.en-zh.yaml
similarity index 79%
rename from lmms_eval/tasks/wmt24pp/wmt24.en-zh.yaml
rename to lmms_eval/tasks/wmt24pp/wmt24pp.en-zh.yaml
index 19c385f0d..ed61e2866 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24.en-zh.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.en-zh.yaml
@@ -30,11 +30,5 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following source text from English to Chinese:\n{source}"
-    post_prompt: ""
-  qwen_vl:
-    pre_prompt: ""
-    post_prompt: " Answer:"
-
-
-
+    pre_prompt: "Translate the following English source text to Chinese.\nEnglish:"
+    post_prompt: "Chinese:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/wmt24pp/wmt24pp.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.yaml
index 3b268340b..7d47c1cc3 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24pp.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.yaml
@@ -1,23 +1,13 @@
 group: wmt24pp
 task:
+  - wmt24pp.en-cs
   - wmt24pp.en-de
+  - wmt24pp.en-es
   - wmt24pp.en-fr
-  - wmt24pp.en-cs
-  - wmt24pp.en-pt
   - wmt24pp.en-it
   - wmt24pp.en-ko
   - wmt24pp.en-nl
+  - wmt24pp.en-pt
   - wmt24pp.en-ru
-  - wmt24pp.en-es
   - wmt24pp.en-zh
-  - wmt24pp.de-en
-  - wmt24pp.fr-en
-  - wmt24pp.cs-en
-  - wmt24pp.pt-en
-  - wmt24pp.it-en
-  - wmt24pp.ko-en
-  - wmt24pp.nl-en
-  - wmt24pp.ru-en
-  - wmt24pp.es-en
-  - wmt24pp.zh-en
 

From 07f628ecfc83f316436f057d09a08867097d4e80 Mon Sep 17 00:00:00 2001
From: amin <amin@unbabel.com>
Date: Mon, 31 Mar 2025 15:33:51 +0000
Subject: [PATCH 071/112] updated wmt24pp tasks

---
 lmms_eval/tasks/wmt24pp/utils.py           | 8 ++++----
 lmms_eval/tasks/wmt24pp/wmt24pp.en-cs.yaml | 4 ++--
 lmms_eval/tasks/wmt24pp/wmt24pp.en-de.yaml | 5 ++---
 lmms_eval/tasks/wmt24pp/wmt24pp.en-es.yaml | 4 ++--
 lmms_eval/tasks/wmt24pp/wmt24pp.en-fr.yaml | 4 ++--
 lmms_eval/tasks/wmt24pp/wmt24pp.en-it.yaml | 4 ++--
 lmms_eval/tasks/wmt24pp/wmt24pp.en-ko.yaml | 4 ++--
 lmms_eval/tasks/wmt24pp/wmt24pp.en-nl.yaml | 8 ++++----
 lmms_eval/tasks/wmt24pp/wmt24pp.en-pt.yaml | 4 ++--
 lmms_eval/tasks/wmt24pp/wmt24pp.en-ru.yaml | 4 ++--
 lmms_eval/tasks/wmt24pp/wmt24pp.en-zh.yaml | 4 ++--
 11 files changed, 26 insertions(+), 27 deletions(-)

diff --git a/lmms_eval/tasks/wmt24pp/utils.py b/lmms_eval/tasks/wmt24pp/utils.py
index bcb258671..c796d9800 100644
--- a/lmms_eval/tasks/wmt24pp/utils.py
+++ b/lmms_eval/tasks/wmt24pp/utils.py
@@ -3,7 +3,7 @@
 
 def process_docs(docs):
     # docs = docs.select(range(10))
-    # docs = docs.filter(lambda x: x["is_bad_source"] != "true")
+    docs = docs.filter(lambda x: x["is_bad_source"] != "true")
     return docs
 
 def doc_to_text(doc, lmms_eval_specific_kwargs):
@@ -24,9 +24,9 @@ def process_results(doc, results):
 
 def aggregate_results(results):
     comet = RefCOMET(model="Unbabel/XCOMET-XL")
-    sources = [res["source"] for res in results]
-    hypotheses = [res["prediction"] for res in results]
-    references = [res["ground_truth"] for res in results]
+    sources = [res["source"].strip() for res in results]
+    hypotheses = [res["prediction"].strip() for res in results]
+    references = [res["ground_truth"].strip() for res in results]
     comet.make_samples(sources, hypotheses, references)
     segments_scores_correct = comet.evaluate(hypotheses, references, sources, gpus=1, batch_size=16).result["segments_scores"]
     results = {
diff --git a/lmms_eval/tasks/wmt24pp/wmt24pp.en-cs.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.en-cs.yaml
index a3ded92cf..5339e0922 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24pp.en-cs.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.en-cs.yaml
@@ -10,8 +10,8 @@ doc_to_text: !function utils.doc_to_text
 doc_to_target: "cs"
 
 generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
+  max_new_tokens: 4096
+  temperature: 0.0
   top_p: 1.0
   num_beams: 1
   do_sample: false
diff --git a/lmms_eval/tasks/wmt24pp/wmt24pp.en-de.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.en-de.yaml
index 6cf76a0ad..3ef235043 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24pp.en-de.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.en-de.yaml
@@ -10,8 +10,8 @@ doc_to_text: !function utils.doc_to_text
 doc_to_target: "de"
 
 generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
+  max_new_tokens: 4096
+  temperature: 0.0
   top_p: 1.0
   num_beams: 1
   do_sample: false
@@ -24,7 +24,6 @@ metric_list:
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
-    eval_model: "Unbabel/XCOMET-XXL"
 
 metadata:
   - version: 0.0
diff --git a/lmms_eval/tasks/wmt24pp/wmt24pp.en-es.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.en-es.yaml
index 73b4b9e91..d26321b34 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24pp.en-es.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.en-es.yaml
@@ -10,8 +10,8 @@ doc_to_text: !function utils.doc_to_text
 doc_to_target: "es"
 
 generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
+  max_new_tokens: 4096
+  temperature: 0.0
   top_p: 1.0
   num_beams: 1
   do_sample: false
diff --git a/lmms_eval/tasks/wmt24pp/wmt24pp.en-fr.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.en-fr.yaml
index f9c8d31a7..afd665304 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24pp.en-fr.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.en-fr.yaml
@@ -11,8 +11,8 @@ doc_to_text: !function utils.doc_to_text
 doc_to_target: "fr"
 
 generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
+  max_new_tokens: 4096
+  temperature: 0.0
   top_p: 1.0
   num_beams: 1
   do_sample: false
diff --git a/lmms_eval/tasks/wmt24pp/wmt24pp.en-it.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.en-it.yaml
index a2b541c2c..2ae40ef03 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24pp.en-it.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.en-it.yaml
@@ -10,8 +10,8 @@ doc_to_text: !function utils.doc_to_text
 doc_to_target: "it"
 
 generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
+  max_new_tokens: 4096
+  temperature: 0.0
   top_p: 1.0
   num_beams: 1
   do_sample: false
diff --git a/lmms_eval/tasks/wmt24pp/wmt24pp.en-ko.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.en-ko.yaml
index 3860025dc..4bfe6dd3e 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24pp.en-ko.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.en-ko.yaml
@@ -10,8 +10,8 @@ doc_to_text: !function utils.doc_to_text
 doc_to_target: "ko"
 
 generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
+  max_new_tokens: 4096
+  temperature: 0.0
   top_p: 1.0
   num_beams: 1
   do_sample: false
diff --git a/lmms_eval/tasks/wmt24pp/wmt24pp.en-nl.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.en-nl.yaml
index f758b3662..3e2f0de9d 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24pp.en-nl.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.en-nl.yaml
@@ -10,8 +10,8 @@ doc_to_text: !function utils.doc_to_text
 doc_to_target: "nl"
 
 generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
+  max_new_tokens: 4096
+  temperature: 0.0
   top_p: 1.0
   num_beams: 1
   do_sample: false
@@ -30,5 +30,5 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following English source text to Dutch.\nEnglish:"
-    post_prompt: "Dutch:"
\ No newline at end of file
+    pre_prompt: "Translate the following English source text to Duetch.\nEnglish:"
+    post_prompt: "Duetch:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/wmt24pp/wmt24pp.en-pt.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.en-pt.yaml
index 9d705c7c2..2d5f10096 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24pp.en-pt.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.en-pt.yaml
@@ -10,8 +10,8 @@ doc_to_text: !function utils.doc_to_text
 doc_to_target: "pt"
 
 generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
+  max_new_tokens: 4096
+  temperature: 0.0
   top_p: 1.0
   num_beams: 1
   do_sample: false
diff --git a/lmms_eval/tasks/wmt24pp/wmt24pp.en-ru.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.en-ru.yaml
index af25b583b..4effafab9 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24pp.en-ru.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.en-ru.yaml
@@ -10,8 +10,8 @@ doc_to_text: !function utils.doc_to_text
 doc_to_target: "ru"
 
 generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
+  max_new_tokens: 4096
+  temperature: 0.0
   top_p: 1.0
   num_beams: 1
   do_sample: false
diff --git a/lmms_eval/tasks/wmt24pp/wmt24pp.en-zh.yaml b/lmms_eval/tasks/wmt24pp/wmt24pp.en-zh.yaml
index ed61e2866..b7965d614 100644
--- a/lmms_eval/tasks/wmt24pp/wmt24pp.en-zh.yaml
+++ b/lmms_eval/tasks/wmt24pp/wmt24pp.en-zh.yaml
@@ -10,8 +10,8 @@ doc_to_text: !function utils.doc_to_text
 doc_to_target: "zh"
 
 generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
+  max_new_tokens: 4096
+  temperature: 0.0
   top_p: 1.0
   num_beams: 1
   do_sample: false

From 766f7520825dda485e216b08b3aa30ffb9fa21e0 Mon Sep 17 00:00:00 2001
From: amin <amin@unbabel.com>
Date: Mon, 31 Mar 2025 15:36:05 +0000
Subject: [PATCH 072/112] added the possibility of defining a custom
 system_prompt for Pangea

---
 lmms_eval/models/pangea.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/lmms_eval/models/pangea.py b/lmms_eval/models/pangea.py
index 88a1e1eb8..76316aa67 100644
--- a/lmms_eval/models/pangea.py
+++ b/lmms_eval/models/pangea.py
@@ -61,6 +61,7 @@ def __init__(
         tie_weights: bool = True,
         truncate_context=False,  # whether to truncate the context in generation, set it False for LLaVA-1.6
         customized_config=None,  # ends in json
+        system_prompt=None,
         **kwargs,
     ) -> None:
         super().__init__()
@@ -143,7 +144,7 @@ def __init__(
             self._rank = 0
             self._world_size = 1
                 
-
+        self.system_prompt = system_prompt
     @property
     def config(self):
         # return the associated transformers.AutoConfig for the given pretrained model.
@@ -296,7 +297,6 @@ def _collate(x):
             # - any OOMs will happen right away rather than near the end
             toks = self.tok_encode(x[0])
             return -len(toks), x[0]
-
         # we group requests by their generation_kwargs,
         # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
         # in the same batch.
@@ -361,6 +361,8 @@ def _collate(x):
                     conv = copy.deepcopy(conv_templates[self.conv_template])
                 else:
                     conv = conv_templates[self.conv_template].copy()
+                if self.system_prompt is not None:
+                    conv.system = "<|im_start|>system\n" + self.system_prompt
                 conv.append_message(conv.roles[0], question)
                 conv.append_message(conv.roles[1], None)
                 prompt_question = conv.get_prompt()
@@ -370,7 +372,7 @@ def _collate(x):
             # preconfigure gen_kwargs with defaults
             gen_kwargs["image_sizes"] = [flattened_visuals[idx].size for idx in range(len(flattened_visuals))]
             if "max_new_tokens" not in gen_kwargs:
-                gen_kwargs["max_new_tokens"] = 1024
+                gen_kwargs["max_new_tokens"] = 4096
             if "temperature" not in gen_kwargs:
                 gen_kwargs["temperature"] = 0
             if "top_p" not in gen_kwargs:
@@ -391,7 +393,7 @@ def _collate(x):
                     pad_token_id=pad_token_ids,
                     images=image_tensor,
                     image_sizes=gen_kwargs["image_sizes"],
-                    do_sample=True if gen_kwargs["temperature"] > 0 else False,
+                    do_sample=gen_kwargs["temperature"] > 0,
                     temperature=gen_kwargs["temperature"],
                     top_p=gen_kwargs["top_p"],
                     num_beams=gen_kwargs["num_beams"],

From 266e9c024abf0bffc7295730ee54e4a0d1104e28 Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Mon, 7 Apr 2025 19:10:34 +0000
Subject: [PATCH 073/112] added arc text-only for multiple languages and aya
 vision bench

---
 lmms_eval/tasks/arc/arc_challenge_all.yaml    |  9 +++
 lmms_eval/tasks/arc/arc_challenge_de.yaml     | 44 ++++++++++
 lmms_eval/tasks/arc/arc_challenge_eng.yaml    | 44 ++++++++++
 lmms_eval/tasks/arc/arc_challenge_es.yaml     | 44 ++++++++++
 lmms_eval/tasks/arc/arc_challenge_fr.yaml     | 44 ++++++++++
 lmms_eval/tasks/arc/arc_challenge_it.yaml     | 44 ++++++++++
 lmms_eval/tasks/arc/arc_challenge_nl.yaml     | 44 ++++++++++
 lmms_eval/tasks/arc/arc_challenge_pt.yaml     | 44 ++++++++++
 lmms_eval/tasks/arc/arc_easy_v2.yaml          | 42 ++++++++++
 lmms_eval/tasks/arc/utils.py                  | 65 +++++++++++++++
 .../aya-vision-bench-gen-cs.yaml              | 29 +++++++
 .../aya-vision-bench-gen-de.yaml              | 29 +++++++
 .../aya-vision-bench-gen-eng.yaml             | 31 +++++++
 .../aya-vision-bench-gen-es.yaml              | 29 +++++++
 .../aya-vision-bench-gen-fr.yaml              | 29 +++++++
 .../aya-vision-bench-gen-it.yaml              | 29 +++++++
 .../aya-vision-bench-gen-ko.yaml              | 29 +++++++
 .../aya-vision-bench-gen-nl.yaml              | 29 +++++++
 .../aya-vision-bench-gen-pt.yaml              | 29 +++++++
 .../aya-vision-bench-gen-ru.yaml              | 29 +++++++
 .../aya-vision-bench-gen-zh.yaml              | 29 +++++++
 .../aya-vision-bench-gen.yaml                 | 14 ++++
 lmms_eval/tasks/aya-vision-bench/utils.py     | 81 +++++++++++++++++++
 23 files changed, 840 insertions(+)
 create mode 100644 lmms_eval/tasks/arc/arc_challenge_all.yaml
 create mode 100644 lmms_eval/tasks/arc/arc_challenge_de.yaml
 create mode 100644 lmms_eval/tasks/arc/arc_challenge_eng.yaml
 create mode 100644 lmms_eval/tasks/arc/arc_challenge_es.yaml
 create mode 100644 lmms_eval/tasks/arc/arc_challenge_fr.yaml
 create mode 100644 lmms_eval/tasks/arc/arc_challenge_it.yaml
 create mode 100644 lmms_eval/tasks/arc/arc_challenge_nl.yaml
 create mode 100644 lmms_eval/tasks/arc/arc_challenge_pt.yaml
 create mode 100644 lmms_eval/tasks/arc/arc_easy_v2.yaml
 create mode 100644 lmms_eval/tasks/arc/utils.py
 create mode 100644 lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-cs.yaml
 create mode 100644 lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-de.yaml
 create mode 100644 lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-eng.yaml
 create mode 100644 lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-es.yaml
 create mode 100644 lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-fr.yaml
 create mode 100644 lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-it.yaml
 create mode 100644 lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-ko.yaml
 create mode 100644 lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-nl.yaml
 create mode 100644 lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-pt.yaml
 create mode 100644 lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-ru.yaml
 create mode 100644 lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-zh.yaml
 create mode 100644 lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen.yaml
 create mode 100644 lmms_eval/tasks/aya-vision-bench/utils.py

diff --git a/lmms_eval/tasks/arc/arc_challenge_all.yaml b/lmms_eval/tasks/arc/arc_challenge_all.yaml
new file mode 100644
index 000000000..b127d4c65
--- /dev/null
+++ b/lmms_eval/tasks/arc/arc_challenge_all.yaml
@@ -0,0 +1,9 @@
+group: arc_challenge_all
+task:
+  - arc_challenge_eng
+  - arc_challenge_de
+  - arc_challenge_es
+  - arc_challenge_fr
+  - arc_challenge_it
+  - arc_challenge_nl
+  - arc_challenge_pt
diff --git a/lmms_eval/tasks/arc/arc_challenge_de.yaml b/lmms_eval/tasks/arc/arc_challenge_de.yaml
new file mode 100644
index 000000000..a86cafb32
--- /dev/null
+++ b/lmms_eval/tasks/arc/arc_challenge_de.yaml
@@ -0,0 +1,44 @@
+tag:
+  - ai2_arc
+task: arc_challenge_de
+dataset_path: LumiOpen/arc_challenge_mt
+dataset_name: de
+output_type: multiple_choice
+test_split: test
+
+
+# doc_to_choice: "{{choices.text}}"
+# doc_to_text: "Question: {{question}}\nAnswer:"
+# doc_to_target: "{{choices.label.index(answerKey)}}"
+# should_decontaminate: true
+# doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
+# metric_list:
+#   - metric: acc
+#     aggregation: mean
+#     higher_is_better: true
+#   - metric: acc_norm
+#     aggregation: mean
+#     higher_is_better: true
+
+
+doc_to_target: "answer"
+process_docs: !function utils.process_docs
+doc_to_text: !function utils.doc_to_text
+doc_to_visual: !function utils.doc_to_visual
+doc_to_choice: !function utils.doc_to_choice
+process_results: !function utils.process_results
+
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Given the following question and four candidate answers (A, B, C and D), choose the best answer text only. No additional text. Answer strictly in German."
+    post_prompt: "Answer:"
+
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+
+metadata:
+  version: 1.0
diff --git a/lmms_eval/tasks/arc/arc_challenge_eng.yaml b/lmms_eval/tasks/arc/arc_challenge_eng.yaml
new file mode 100644
index 000000000..c63e3a0f1
--- /dev/null
+++ b/lmms_eval/tasks/arc/arc_challenge_eng.yaml
@@ -0,0 +1,44 @@
+tag:
+  - ai2_arc
+task: arc_challenge_eng
+dataset_path: allenai/ai2_arc
+dataset_name: ARC-Challenge
+output_type: multiple_choice
+test_split: test
+
+
+# doc_to_choice: "{{choices.text}}"
+# doc_to_text: "Question: {{question}}\nAnswer:"
+# doc_to_target: "{{choices.label.index(answerKey)}}"
+# should_decontaminate: true
+# doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
+# metric_list:
+#   - metric: acc
+#     aggregation: mean
+#     higher_is_better: true
+#   - metric: acc_norm
+#     aggregation: mean
+#     higher_is_better: true
+
+
+doc_to_target: "answer"
+process_docs: !function utils.process_docs
+doc_to_text: !function utils.doc_to_text
+doc_to_visual: !function utils.doc_to_visual
+doc_to_choice: !function utils.doc_to_choice
+process_results: !function utils.process_results
+
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Given the following question and four candidate answers (A, B, C and D), choose the best answer text only. No additional text. Answer strictly in English."
+    post_prompt: "Answer:"
+
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+
+metadata:
+  version: 1.0
diff --git a/lmms_eval/tasks/arc/arc_challenge_es.yaml b/lmms_eval/tasks/arc/arc_challenge_es.yaml
new file mode 100644
index 000000000..2fa7e5120
--- /dev/null
+++ b/lmms_eval/tasks/arc/arc_challenge_es.yaml
@@ -0,0 +1,44 @@
+tag:
+  - ai2_arc
+task: arc_challenge_es
+dataset_path: LumiOpen/arc_challenge_mt
+dataset_name: es
+output_type: multiple_choice
+test_split: test
+
+
+# doc_to_choice: "{{choices.text}}"
+# doc_to_text: "Question: {{question}}\nAnswer:"
+# doc_to_target: "{{choices.label.index(answerKey)}}"
+# should_decontaminate: true
+# doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
+# metric_list:
+#   - metric: acc
+#     aggregation: mean
+#     higher_is_better: true
+#   - metric: acc_norm
+#     aggregation: mean
+#     higher_is_better: true
+
+
+doc_to_target: "answer"
+process_docs: !function utils.process_docs
+doc_to_text: !function utils.doc_to_text
+doc_to_visual: !function utils.doc_to_visual
+doc_to_choice: !function utils.doc_to_choice
+process_results: !function utils.process_results
+
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Given the following question and four candidate answers (A, B, C and D), choose the best answer text only. No additional text. Answer strictly in Spanish."
+    post_prompt: "Answer:"
+
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+
+metadata:
+  version: 1.0
diff --git a/lmms_eval/tasks/arc/arc_challenge_fr.yaml b/lmms_eval/tasks/arc/arc_challenge_fr.yaml
new file mode 100644
index 000000000..6cf49a768
--- /dev/null
+++ b/lmms_eval/tasks/arc/arc_challenge_fr.yaml
@@ -0,0 +1,44 @@
+tag:
+  - ai2_arc
+task: arc_challenge_fr
+dataset_path: LumiOpen/arc_challenge_mt
+dataset_name: fr
+output_type: multiple_choice
+test_split: test
+
+
+# doc_to_choice: "{{choices.text}}"
+# doc_to_text: "Question: {{question}}\nAnswer:"
+# doc_to_target: "{{choices.label.index(answerKey)}}"
+# should_decontaminate: true
+# doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
+# metric_list:
+#   - metric: acc
+#     aggregation: mean
+#     higher_is_better: true
+#   - metric: acc_norm
+#     aggregation: mean
+#     higher_is_better: true
+
+
+doc_to_target: "answer"
+process_docs: !function utils.process_docs
+doc_to_text: !function utils.doc_to_text
+doc_to_visual: !function utils.doc_to_visual
+doc_to_choice: !function utils.doc_to_choice
+process_results: !function utils.process_results
+
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Given the following question and four candidate answers (A, B, C and D), choose the best answer text only. No additional text. Answer strictly in French."
+    post_prompt: "Answer:"
+
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+
+metadata:
+  version: 1.0
diff --git a/lmms_eval/tasks/arc/arc_challenge_it.yaml b/lmms_eval/tasks/arc/arc_challenge_it.yaml
new file mode 100644
index 000000000..ba5d53ef0
--- /dev/null
+++ b/lmms_eval/tasks/arc/arc_challenge_it.yaml
@@ -0,0 +1,44 @@
+tag:
+  - ai2_arc
+task: arc_challenge_it
+dataset_path: LumiOpen/arc_challenge_mt
+dataset_name: it
+output_type: multiple_choice
+test_split: test
+
+
+# doc_to_choice: "{{choices.text}}"
+# doc_to_text: "Question: {{question}}\nAnswer:"
+# doc_to_target: "{{choices.label.index(answerKey)}}"
+# should_decontaminate: true
+# doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
+# metric_list:
+#   - metric: acc
+#     aggregation: mean
+#     higher_is_better: true
+#   - metric: acc_norm
+#     aggregation: mean
+#     higher_is_better: true
+
+
+doc_to_target: "answer"
+process_docs: !function utils.process_docs
+doc_to_text: !function utils.doc_to_text
+doc_to_visual: !function utils.doc_to_visual
+doc_to_choice: !function utils.doc_to_choice
+process_results: !function utils.process_results
+
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Given the following question and four candidate answers (A, B, C and D), choose the best answer text only. No additional text. Answer strictly in Italian."
+    post_prompt: "Answer:"
+
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+
+metadata:
+  version: 1.0
diff --git a/lmms_eval/tasks/arc/arc_challenge_nl.yaml b/lmms_eval/tasks/arc/arc_challenge_nl.yaml
new file mode 100644
index 000000000..298434f23
--- /dev/null
+++ b/lmms_eval/tasks/arc/arc_challenge_nl.yaml
@@ -0,0 +1,44 @@
+tag:
+  - ai2_arc
+task: arc_challenge_nl
+dataset_path: LumiOpen/arc_challenge_mt
+dataset_name: nl
+output_type: multiple_choice
+test_split: test
+
+
+# doc_to_choice: "{{choices.text}}"
+# doc_to_text: "Question: {{question}}\nAnswer:"
+# doc_to_target: "{{choices.label.index(answerKey)}}"
+# should_decontaminate: true
+# doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
+# metric_list:
+#   - metric: acc
+#     aggregation: mean
+#     higher_is_better: true
+#   - metric: acc_norm
+#     aggregation: mean
+#     higher_is_better: true
+
+
+doc_to_target: "answer"
+process_docs: !function utils.process_docs
+doc_to_text: !function utils.doc_to_text
+doc_to_visual: !function utils.doc_to_visual
+doc_to_choice: !function utils.doc_to_choice
+process_results: !function utils.process_results
+
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Given the following question and four candidate answers (A, B, C and D), choose the best answer text only. No additional text. Answer strictly in Dutch."
+    post_prompt: "Answer:"
+
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+
+metadata:
+  version: 1.0
diff --git a/lmms_eval/tasks/arc/arc_challenge_pt.yaml b/lmms_eval/tasks/arc/arc_challenge_pt.yaml
new file mode 100644
index 000000000..6b9f20df0
--- /dev/null
+++ b/lmms_eval/tasks/arc/arc_challenge_pt.yaml
@@ -0,0 +1,44 @@
+tag:
+  - ai2_arc
+task: arc_challenge_pt
+dataset_path: LumiOpen/arc_challenge_mt
+dataset_name: pt
+output_type: multiple_choice
+test_split: test
+
+
+# doc_to_choice: "{{choices.text}}"
+# doc_to_text: "Question: {{question}}\nAnswer:"
+# doc_to_target: "{{choices.label.index(answerKey)}}"
+# should_decontaminate: true
+# doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
+# metric_list:
+#   - metric: acc
+#     aggregation: mean
+#     higher_is_better: true
+#   - metric: acc_norm
+#     aggregation: mean
+#     higher_is_better: true
+
+
+doc_to_target: "answer"
+process_docs: !function utils.process_docs
+doc_to_text: !function utils.doc_to_text
+doc_to_visual: !function utils.doc_to_visual
+doc_to_choice: !function utils.doc_to_choice
+process_results: !function utils.process_results
+
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Given the following question and four candidate answers (A, B, C and D), choose the best answer text only. No additional text. Answer strictly in Portuguese."
+    post_prompt: "Answer:"
+
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+
+metadata:
+  version: 1.0
diff --git a/lmms_eval/tasks/arc/arc_easy_v2.yaml b/lmms_eval/tasks/arc/arc_easy_v2.yaml
new file mode 100644
index 000000000..6aa7b9e29
--- /dev/null
+++ b/lmms_eval/tasks/arc/arc_easy_v2.yaml
@@ -0,0 +1,42 @@
+tag:
+  - ai2_arc
+task: arc_easy_v2
+dataset_path: allenai/ai2_arc
+dataset_name: ARC-Easy
+output_type: multiple_choice
+test_split: test
+
+
+# doc_to_choice: "{{choices.text}}"
+# doc_to_text: "Question: {{question}}\nAnswer:"
+# doc_to_target: "{{choices.label.index(answerKey)}}"
+# should_decontaminate: true
+# doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
+# metric_list:
+#   - metric: acc
+#     aggregation: mean
+#     higher_is_better: true
+#   - metric: acc_norm
+#     aggregation: mean
+#     higher_is_better: true
+
+doc_to_target: ""
+process_docs: !function utils.process_docs
+doc_to_text: !function utils.doc_to_text
+doc_to_visual: !function utils.doc_to_visual
+doc_to_choice: !function utils.doc_to_choice
+process_results: !function utils.process_results
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer with the correct text only. No additional text.\n"
+    post_prompt: "Answer:"
+
+
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
+    higher_is_better: true
+
+metadata:
+  version: 1.0
diff --git a/lmms_eval/tasks/arc/utils.py b/lmms_eval/tasks/arc/utils.py
new file mode 100644
index 000000000..eb151352e
--- /dev/null
+++ b/lmms_eval/tasks/arc/utils.py
@@ -0,0 +1,65 @@
+
+import base64
+from PIL import Image
+from io import BytesIO
+import numpy as np
+import re
+from loguru import logger
+
+def process_docs(docs):
+    """
+    Process documents...
+    """
+    # logger.info(f"processing docs")
+    # Process images in place
+    # docs = docs.select(range(5)) # filter out some samples!
+    return docs
+
+# doc_to_text: "Question: {{question}}\nAnswer:"
+# doc_to_target: "{{choices.label.index(answerKey)}}"
+# doc_to_choice: "{{choices.text}}"
+
+def doc_to_visual(doc):
+    return []
+
+
+def doc_to_text(doc,lmms_eval_specific_kwargs=None ):
+    question = doc["question"]
+    choices = doc["choices"]
+    choices_text = choices["text"]
+    choices_label = choices["label"]
+    choices = "\n".join([f"{label}: {text}" for label, text in zip(choices_label, choices_text)])
+    my_text = f"Question: {question}\n{choices}\n"
+    if 'pre_prompt' in lmms_eval_specific_kwargs:
+        pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+        my_text = f"{pre_prompt}\n{my_text}"
+    else:
+        pre_prompt = ""
+    if 'post_prompt' in lmms_eval_specific_kwargs:
+        post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+        my_text = f"{my_text}\n{post_prompt}"
+    else:
+        post_prompt = ""
+    return f"{my_text}"
+
+
+def doc_to_choice(doc,lmms_eval_specific_kwargs=None ):
+    choices = doc["choices"]
+    choice_list = [f"{label}: {text}" for label, text in zip(choices["label"], choices["text"])]
+    # choice_list = choices["text"]
+    return choice_list
+
+
+def process_results(doc, results):
+    losses = [res[0] for res in results]
+    pred_index = losses.index(min(losses))
+    pred_letter = doc["choices"]["label"][pred_index]
+    answer = doc["answerKey"]
+    return {"results": {"id": doc["id"], "question": doc["question"], "prediction": pred_letter, "ground_truth": answer} }
+
+
+def aggregate_results(results):
+    preds = [result["prediction"] for result in results]
+    gts = [result["ground_truth"] for result in results]
+    acc = sum([pred == gt for pred, gt in zip(preds, gts)]) / len(preds)
+    return {"acc": acc}
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-cs.yaml b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-cs.yaml
new file mode 100644
index 000000000..25368377b
--- /dev/null
+++ b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-cs.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: ces_Latn
+task: aya-vision-bench-gen-cs
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-de.yaml b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-de.yaml
new file mode 100644
index 000000000..59cc4208c
--- /dev/null
+++ b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-de.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: deu_Latn
+task: aya-vision-bench-gen-de
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-eng.yaml b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-eng.yaml
new file mode 100644
index 000000000..d344b49b9
--- /dev/null
+++ b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-eng.yaml
@@ -0,0 +1,31 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: eng_Latn
+task: aya-vision-bench-gen-eng
+test_split: test
+output_type: generate_until
+
+process_docs: !function utils.process_docs
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-es.yaml b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-es.yaml
new file mode 100644
index 000000000..d31c4f996
--- /dev/null
+++ b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-es.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: spa_Latn
+task: aya-vision-bench-gen-es
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-fr.yaml b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-fr.yaml
new file mode 100644
index 000000000..34ffa0b10
--- /dev/null
+++ b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-fr.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: fra_Latn
+task: aya-vision-bench-gen-fr
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-it.yaml b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-it.yaml
new file mode 100644
index 000000000..12c86333f
--- /dev/null
+++ b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-it.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: ita_Latn
+task: aya-vision-bench-gen-it
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-ko.yaml b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-ko.yaml
new file mode 100644
index 000000000..620286bc2
--- /dev/null
+++ b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-ko.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: kor_Hang
+task: aya-vision-bench-gen-ko
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-nl.yaml b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-nl.yaml
new file mode 100644
index 000000000..6267d8c18
--- /dev/null
+++ b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-nl.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: nld_Latn
+task: aya-vision-bench-gen-nl
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-pt.yaml b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-pt.yaml
new file mode 100644
index 000000000..afde9cb63
--- /dev/null
+++ b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-pt.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: por_Latn
+task: aya-vision-bench-gen-pt
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-ru.yaml b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-ru.yaml
new file mode 100644
index 000000000..d2c30f97e
--- /dev/null
+++ b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-ru.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: rus_Cyrl
+task: aya-vision-bench-gen-ru
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-zh.yaml b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-zh.yaml
new file mode 100644
index 000000000..8ebb6fb34
--- /dev/null
+++ b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-zh.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: zho_Hans
+task: aya-vision-bench-gen-zh
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen.yaml b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen.yaml
new file mode 100644
index 000000000..ea6402f48
--- /dev/null
+++ b/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen.yaml
@@ -0,0 +1,14 @@
+group: aya-vision-bench-gen
+task:
+  - aya-vision-bench-gen-eng
+  - aya-vision-bench-gen-de
+  - aya-vision-bench-gen-es
+  - aya-vision-bench-gen-fr
+  - aya-vision-bench-gen-it
+  - aya-vision-bench-gen-ko
+  - aya-vision-bench-gen-zh
+  - aya-vision-bench-gen-nl
+  - aya-vision-bench-gen-pt
+  - aya-vision-bench-gen-ru
+  # - aya-vision-bench-gen-cs
+
diff --git a/lmms_eval/tasks/aya-vision-bench/utils.py b/lmms_eval/tasks/aya-vision-bench/utils.py
new file mode 100644
index 000000000..abf3f767a
--- /dev/null
+++ b/lmms_eval/tasks/aya-vision-bench/utils.py
@@ -0,0 +1,81 @@
+import base64
+from PIL import Image
+from io import BytesIO
+import numpy as np
+import re
+import json
+import os
+from loguru import logger
+
+
+def base64_to_bytes(base64_string):
+    # Remove the header if it exists (e.g., "data:image/jpeg;base64,")
+    if "base64," in base64_string:
+        base64_string = base64_string.split("base64,")[1]
+    # Decode base64 string to bytes
+    img_bytes = base64.b64decode(base64_string)
+    return img_bytes
+
+def process_docs(docs):
+    """
+    Process documents...
+    """
+    # logger.info(f"processing docs")
+    # Process images in place
+    docs = docs.select(range(5)) # filter out some samples!
+    return docs
+
+
+def gen_doc_to_visual(doc):
+    image = doc['image'][0]
+    image = image.convert('RGB')
+    return [image]
+
+
+def gen_doc_to_text(doc,lmms_eval_specific_kwargs=None ):
+    question = doc["question"]
+    if 'pre_prompt' in lmms_eval_specific_kwargs:
+        pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    else:
+        pre_prompt = ""
+    if 'post_prompt' in lmms_eval_specific_kwargs:
+        post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+    else:
+        post_prompt = ""
+    return f"{pre_prompt}Question: {question}\n{post_prompt}"
+
+
+# def doc_to_choice(doc,lmms_eval_specific_kwargs=None ):
+#     choices = doc["choices"]
+#     choice_list = [f"{label}: {text}" for label, text in zip(choices["label"], choices["text"])]
+#     choice_list = choices["text"]
+#     return choice_list
+
+# def doc_to_target(doc,lmms_eval_specific_kwargs=None ):
+#     choices = doc["choices"]
+#     answerKey = doc["answerKey"]
+#     import pdb; pdb.set_trace()
+#     return choices["label"].index(answerKey)
+
+def gen_process_results(doc, results):
+
+    generated_text = [res[0] for res in results]
+    
+    return {"results": {
+        "id": doc["index"],
+        "image": doc["image"],
+        "question": doc["question"],
+        "image_category": doc["image_source_category"],
+        "prediction": generated_text 
+        } 
+    }
+
+
+# def save_result_to_cache(doc, round_res, previous_round_info, save_dir):
+#     save_dict = dict(
+#         sample_id=doc["index"],
+#         question=doc["question"],
+#         round_res=round_res,
+#     )
+#     save_dict.update(previous_round_info)
+#     json.dump(save_dict, open(os.path.join(save_dir, f"{save_dict['sample_id']}.json"), "w"), indent=4)

From eaced6d77c0db9573da35d957f68551e3ccd1624 Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Mon, 7 Apr 2025 21:07:52 +0000
Subject: [PATCH 074/112] qwen2.5 loglikelihood added

---
 lmms_eval/models/qwen2_5_vl.py | 96 +++++++++++++++++++++++++++++++++-
 1 file changed, 95 insertions(+), 1 deletion(-)

diff --git a/lmms_eval/models/qwen2_5_vl.py b/lmms_eval/models/qwen2_5_vl.py
index f65beb09e..4c83f58ae 100644
--- a/lmms_eval/models/qwen2_5_vl.py
+++ b/lmms_eval/models/qwen2_5_vl.py
@@ -153,8 +153,102 @@ def world_size(self):
         return self._world_size
 
     def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
-        raise NotImplementedError("Loglikelihood is not implemented for Qwen2.5_VL")
+        # raise NotImplementedError("Loglikelihood is not implemented for Qwen2.5_VL")
+        res = []
+        conversation = []
+        pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
+        for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
+            # encode, pad, and truncate contexts for this batch
+            if type(doc_to_target) == str:
+                continuation = doc_to_target
+            else:
+                continuation = doc_to_target(self.task_dict[task][split][doc_id])
+            visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
+            visuals = self.flatten(visuals)
+            image_sizes = [[visual.size[0], visual.size[1]] for visual in visuals]
+            message = [{"role": "system", "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."}]
+
+            if len(visuals) > 0:
+                visual = visuals[i] if i < len(visuals) else None
+                if isinstance(visual, str) and visual.endswith((".mp4", ".avi", ".mov")):  # Video file
+                    if self.use_custom_video_loader:
+                        visual = read_video_pyav_base64(visual, num_frm=self.max_num_frames, fps=self.fps, img_format="JPEG", max_image_size=self.max_image_size)
+                        image_contents = list(map(lambda x: f"data:image/jpeg;base64,{x}", visual))
+                        message.append({"role": "user", "content": [{"type": "video", "video": image_contents}, {"type": "text", "text": context}]})
+                    else:
+                        vr = decord.VideoReader(visual)
+                        first_frame = vr[0].asnumpy()
+                        height, width = first_frame.shape[:2]
+                        # max_pixels = height * width
+                        message.append({"role": "user", "content": [{"type": "video", "video": visual, "max_pixels": 360 * 420}, {"type": "text", "text": context}]})
+                elif isinstance(visual, Image.Image):  # Single image
+                    base64_image = visual.convert("RGB")
+                    buffer = BytesIO()
+                    base64_image.save(buffer, format="JPEG")
+                    base64_bytes = base64.b64encode(buffer.getvalue())
+                    base64_string = base64_bytes.decode("utf-8")
+                    message.append({"role": "user", "content": [{"type": "image", "image": f"data:image/jpeg;base64,{base64_string}"}, {"type": "text", "text": context}]})
+                elif isinstance(visual, (list, tuple)) and all(isinstance(v, Image.Image) for v in visual):  # Multiple images
+                    image_content = []
+                    for v in visual:
+                        base64_image = v.convert("RGB")
+                        buffer = BytesIO()
+                        base64_image.save(buffer, format="JPEG")
+                        base64_bytes = base64.b64encode(buffer.getvalue())
+                        base64_string = base64_bytes.decode("utf-8")
+                        image_content.append({"type": "image", "image": f"data:image/jpeg;base64,{base64_string}"})
+                    message.append({"role": "user", "content": image_content + [{"type": "text", "text": contexts}]})
+                else:
+                    message.append({"role": "user", "content": [{"type": "text", "text": contexts}]})
+            else:
+                message.append({"role": "user", "content": [{"type": "text", "text": contexts}]})
+
+            
+            text = self.processor.apply_chat_template(message, tokenize=False, add_generation_prompt=True)
+            image_inputs, video_inputs = process_vision_info(message)
+            context_inputs = self.processor(
+                text=[text],
+                images=image_inputs,
+                videos=video_inputs,
+                # fps=self.fps,
+                padding=True,
+                return_tensors="pt",
+            )
+            if self.device_map == "auto":
+                context_inputs = context_inputs.to("cuda")
+            else:
+                context_inputs = context_inputs.to(self.device)
+
+            message.append({"role": "assistant", "content": continuation})
+            text_output = self.processor.apply_chat_template(message, tokenize=False, add_generation_prompt=False)
+            image_inputs, video_inputs = process_vision_info(message)
+            inputs = self.processor(
+                text=[text_output],
+                images=image_inputs,
+                videos=video_inputs,
+                padding=True,
+                return_tensors="pt",
+            )
+            pad_token_id = self.tokenizer.pad_token_id
+            labels = inputs.input_ids.clone()
+            labels[0, : context_inputs.input_ids.shape[1]] = -100
+
+            with torch.inference_mode(): 
+                outputs = self.model(**inputs,labels=labels)
+            
+            loss = outputs["loss"]
+            # loss = torch.exp(loss)
+            logits = outputs["logits"]
+            greedy_tokens = logits.argmax(dim=-1)
+            cont_toks = inputs.input_ids[:, context_inputs.input_ids.shape[1] :]  # [1, seq]
+            greedy_tokens = greedy_tokens[:, context_inputs.input_ids.shape[1] : inputs.input_ids.shape[1]]  # [1, seq]
+            max_equal = (greedy_tokens == cont_toks).all()
+            res.append((float(loss.item()), bool(max_equal)))
+            pbar.update(1)
+        pbar.close()
+        return res
 
+            
     def flatten(self, input):
         new_list = []
         for i in input:

From a37231f5d43ef8a50fab0763836a9c0454179c4d Mon Sep 17 00:00:00 2001
From: amin <amin@unbabel.com>
Date: Tue, 8 Apr 2025 09:00:57 +0000
Subject: [PATCH 075/112] updated llava_hf to work with text-only tasks and not
 throw errors if visuals is None.

---
 lmms_eval/models/llava_hf.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/lmms_eval/models/llava_hf.py b/lmms_eval/models/llava_hf.py
index 935194ae6..03ab13c1a 100644
--- a/lmms_eval/models/llava_hf.py
+++ b/lmms_eval/models/llava_hf.py
@@ -202,11 +202,13 @@ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
             else:
                 continuation = doc_to_target(self.task_dict[task][split][doc_id])
             visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
-            visuals = self.flatten(visuals)
-
-            image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visuals)
-            image_tokens = " ".join(image_tokens)
-            context = f"{image_tokens}\n{context}"
+            if visuals != [None]:
+                visuals = self.flatten(visuals)
+                image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visuals)
+                image_tokens = " ".join(image_tokens)
+                context = f"{image_tokens}\n{context}"
+            else:
+                visuals = None
             # Apply chat template
             messages = [{"role": "user", "content": context}, {"role": "assistant", "content": continuation}]
             if self.chat_template is not None:

From c475b5d3099812861e45d603065d4f025f91b0d6 Mon Sep 17 00:00:00 2001
From: amin <amin@unbabel.com>
Date: Tue, 8 Apr 2025 09:02:31 +0000
Subject: [PATCH 076/112] temporary changes to llava.py

---
 lmms_eval/models/llava.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lmms_eval/models/llava.py b/lmms_eval/models/llava.py
index 1904e4155..c725dbf64 100755
--- a/lmms_eval/models/llava.py
+++ b/lmms_eval/models/llava.py
@@ -65,7 +65,7 @@ def __init__(
         tie_weights: bool = True,
         truncate_context=False,  # whether to truncate the context in generation, set it False for LLaVA-1.6
         customized_config=None,  # ends in json
-        system_prompt=None,
+        system_prompt="<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.",
         **kwargs,
     ) -> None:
         super().__init__()
@@ -211,6 +211,7 @@ def tok_decode(self, tokens):
 
     def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
         # TODO
+        breakpoint()
         res = []
         pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
         for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
@@ -249,9 +250,12 @@ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
                 conv = copy.deepcopy(conv_templates[self.conv_template])
             else:
                 conv = conv_templates[self.conv_template].copy()
+            if self.system_prompt and conv.system:
+                conv.system = self.system_prompt
             conv.append_message(conv.roles[0], prompts_input)
             conv.append_message(conv.roles[1], None)
             prompt = conv.get_prompt()
+
             pad_token_id = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
             contxt_id = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
             # Add the answer of the second role

From e8e30f697f92dced342403bcd6bacb7b48493e62 Mon Sep 17 00:00:00 2001
From: amin <amin@unbabel.com>
Date: Tue, 8 Apr 2025 18:01:39 +0000
Subject: [PATCH 077/112] added loglikelihood to Aya model

---
 lmms_eval/models/aya.py | 75 +++++++++++++++++++++++++++++++++++++++--
 1 file changed, 73 insertions(+), 2 deletions(-)

diff --git a/lmms_eval/models/aya.py b/lmms_eval/models/aya.py
index feed9b589..8fa2f1284 100644
--- a/lmms_eval/models/aya.py
+++ b/lmms_eval/models/aya.py
@@ -8,6 +8,9 @@
 from lmms_eval.api.model import lmms
 from lmms_eval.api.registry import register_model
 
+import torch.nn.functional as F
+import numpy as np
+
 import io
 import base64
 
@@ -149,8 +152,76 @@ def tok_decode(self, tokens):
         return self.tokenizer.decode(tokens)
 
     def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
-        """Not implemented for NVLM model."""
-        raise NotImplementedError("Loglikelihood is not implemented for NVLM model")
+        res = []
+        pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
+        for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
+            # encode, pad, and truncate contexts for this batch
+            if type(doc_to_target) == str:
+                continuation = doc_to_target
+            else:
+                continuation = doc_to_target(self.task_dict[task][split][doc_id])
+            visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
+            
+            if visuals != [None]:
+                visuals = self.flatten(visuals)
+                image_sizes = [[visual.size[0], visual.size[1]] for visual in visuals]
+                image_urls = [self.get_image_url(v) for v in visuals]
+            else:
+                image_urls = None
+
+            prompts_input = contexts[0] if isinstance(contexts, list) else contexts
+
+            # create chat object
+            message = [
+                {"role": "user",
+                 "content": [
+                     {"type": "text", "text": prompts_input
+                      }]}]
+            if image_urls is not None:
+                message[0]["content"] += [{"type": "image", "url": image_url} for image_url in image_urls]
+
+            if self.add_system_prompt is not None:
+                message.insert(0, {"role": "system", 
+                                   "content": [{"type": "text", "text": self.add_system_prompt}]})
+            
+            # Here we haven't appended the continuation yet, so we can use the original message to get the logits of the full input
+            context_id = self._processor.apply_chat_template(message, padding=True, add_generation_prompt=False, tokenize=True, return_dict=True, return_tensors="pt").to(self._model.device)
+            
+            # Now we append the continuation to the message to get the full input
+            message.append({"role": "assistant", "content": [{"type": "text", "text": continuation.strip()}]})
+            input = self._processor.apply_chat_template(message, padding=True, add_generation_prompt=False, tokenize=True, return_dict=True, return_tensors="pt").to(self._model.device)            
+            labels = input["input_ids"].clone()
+
+            
+            # NOTE: The following solution is just a workaround for the fact that we need to mask the extra tokens in the labels
+            """
+            Context part no need to calculate for loss. That's why we set the labels to -100 for the initial context part
+            The answer part is with the following format:
+            '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|><|START_RESPONSE|>ANSWER<|END_RESPONSE|><|END_OF_TURN_TOKEN|>'
+            So, we have 3 tokens before the answer (<|START_OF_TURN_TOKEN|>, <|CHATBOT_TOKEN|>, <|START_RESPONSE|>)
+            and 2 tokens after the answer (<|END_RESPONSE|>, <|END_OF_TURN_TOKEN|>)
+            We need to set the labels for the answer part to -100, so that they are ignored in the loss calculation
+            """
+            labels[0, : context_id["input_ids"].shape[1]+3] = -100
+            labels[0, -2:] = -100  # last token is also ignored
+            try:
+                # NOTE: We have to set `add_generation_prompt` to False here, because we are not generating text, but rather calculating the log likelihood of the continuation.
+                with torch.inference_mode():
+                    outputs = self._model(**input, labels=labels, add_generation_prompt=False)
+                loss = outputs["loss"]
+                logits = outputs["logits"]
+                greedy_tokens = logits.argmax(dim=-1)
+                cont_toks = input["input_ids"][:, context_id["input_ids"].shape[1] :]  # [1, seq]
+                greedy_tokens = greedy_tokens[:, context_id["input_ids"].shape[1] : input["input_ids"].shape[1]]  # [1, seq]
+                max_equal = (greedy_tokens == cont_toks).all()
+                result = (float(loss.item()), bool(max_equal))
+            except Exception as e:
+                eval_logger.error(f"Error {e} in generating")
+                result = ""
+            res.append(result)
+            pbar.update(1)
+        pbar.close()
+        return res
 
     def flatten(self, input):
         """Flatten a nested list."""

From b18d8ed2ddf3451fa57982becddd0f444aaba21c Mon Sep 17 00:00:00 2001
From: amin <amin@unbabel.com>
Date: Tue, 8 Apr 2025 18:06:07 +0000
Subject: [PATCH 078/112] added a small comment to aya model

---
 lmms_eval/models/aya.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lmms_eval/models/aya.py b/lmms_eval/models/aya.py
index 8fa2f1284..48d9c1924 100644
--- a/lmms_eval/models/aya.py
+++ b/lmms_eval/models/aya.py
@@ -188,6 +188,7 @@ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
             context_id = self._processor.apply_chat_template(message, padding=True, add_generation_prompt=False, tokenize=True, return_dict=True, return_tensors="pt").to(self._model.device)
             
             # Now we append the continuation to the message to get the full input
+            # NOTE: We are stripping the answer here so that there is no leading space in the answer
             message.append({"role": "assistant", "content": [{"type": "text", "text": continuation.strip()}]})
             input = self._processor.apply_chat_template(message, padding=True, add_generation_prompt=False, tokenize=True, return_dict=True, return_tensors="pt").to(self._model.device)            
             labels = input["input_ids"].clone()

From 03cf759b927ba209c18489c234db7d8e9445df2c Mon Sep 17 00:00:00 2001
From: amin <amin@unbabel.com>
Date: Tue, 8 Apr 2025 18:14:53 +0000
Subject: [PATCH 079/112] changed slightly the loglikelihood function of llava
 to only consider the answer and not the EOS and new line tokens

---
 lmms_eval/models/llava.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/lmms_eval/models/llava.py b/lmms_eval/models/llava.py
index c725dbf64..2ae9babad 100755
--- a/lmms_eval/models/llava.py
+++ b/lmms_eval/models/llava.py
@@ -211,7 +211,6 @@ def tok_decode(self, tokens):
 
     def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
         # TODO
-        breakpoint()
         res = []
         pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
         for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
@@ -259,13 +258,19 @@ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
             pad_token_id = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
             contxt_id = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
             # Add the answer of the second role
-            conv.messages[1][1] = continuation
+            # NOTE: We are stripping the answer here so that there is no leading space in the answer
+            conv.messages[1][1] = continuation.strip()
 
             prompt = conv.get_prompt()
             input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
             labels = input_ids.clone()
             # Context part no need to calculate for loss
             labels[0, : contxt_id.shape[1]] = -100
+            
+            # NOTE: We mask the last two tokens in the labels, which are the EOS and the new line tokens.
+            # This is to avoid calculating loss on these tokens and have a more comparable loss to the other models.
+            labels[0, -2:] = -100
+            
             with torch.inference_mode():
                 outputs = self.model(input_ids=input_ids, labels=labels, images=image, use_cache=True, image_sizes=image_sizes)
             loss = outputs["loss"]

From fa296ead0b6629dd841a26548199974140d5071f Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Thu, 10 Apr 2025 18:46:22 +0000
Subject: [PATCH 080/112] added judges in aya vision

---
 lmms_eval/tasks/aya-vision-bench/utils.py     |  81 ------
 .../aya-vision-bench-gen-cs.yaml              |   0
 .../aya-vision-bench-gen-de.yaml              |   0
 .../aya-vision-bench-gen-eng.yaml             |   7 +-
 .../aya-vision-bench-gen-es.yaml              |   0
 .../aya-vision-bench-gen-fr.yaml              |   0
 .../aya-vision-bench-gen-it.yaml              |   0
 .../aya-vision-bench-gen-ko.yaml              |   0
 .../aya-vision-bench-gen-nl.yaml              |   0
 .../aya-vision-bench-gen-pt.yaml              |   0
 .../aya-vision-bench-gen-ru.yaml              |   0
 .../aya-vision-bench-gen-zh.yaml              |   0
 .../aya-vision-bench-gen.yaml                 |   0
 .../eval_with_judge_template.yaml             |  13 +
 .../tasks/ayavisionbench/judge_templates.py   |  39 +++
 lmms_eval/tasks/ayavisionbench/utils.py       | 231 ++++++++++++++++++
 16 files changed, 288 insertions(+), 83 deletions(-)
 delete mode 100644 lmms_eval/tasks/aya-vision-bench/utils.py
 rename lmms_eval/tasks/{aya-vision-bench => ayavisionbench}/aya-vision-bench-gen-cs.yaml (100%)
 rename lmms_eval/tasks/{aya-vision-bench => ayavisionbench}/aya-vision-bench-gen-de.yaml (100%)
 rename lmms_eval/tasks/{aya-vision-bench => ayavisionbench}/aya-vision-bench-gen-eng.yaml (87%)
 rename lmms_eval/tasks/{aya-vision-bench => ayavisionbench}/aya-vision-bench-gen-es.yaml (100%)
 rename lmms_eval/tasks/{aya-vision-bench => ayavisionbench}/aya-vision-bench-gen-fr.yaml (100%)
 rename lmms_eval/tasks/{aya-vision-bench => ayavisionbench}/aya-vision-bench-gen-it.yaml (100%)
 rename lmms_eval/tasks/{aya-vision-bench => ayavisionbench}/aya-vision-bench-gen-ko.yaml (100%)
 rename lmms_eval/tasks/{aya-vision-bench => ayavisionbench}/aya-vision-bench-gen-nl.yaml (100%)
 rename lmms_eval/tasks/{aya-vision-bench => ayavisionbench}/aya-vision-bench-gen-pt.yaml (100%)
 rename lmms_eval/tasks/{aya-vision-bench => ayavisionbench}/aya-vision-bench-gen-ru.yaml (100%)
 rename lmms_eval/tasks/{aya-vision-bench => ayavisionbench}/aya-vision-bench-gen-zh.yaml (100%)
 rename lmms_eval/tasks/{aya-vision-bench => ayavisionbench}/aya-vision-bench-gen.yaml (100%)
 create mode 100644 lmms_eval/tasks/ayavisionbench/eval_with_judge_template.yaml
 create mode 100755 lmms_eval/tasks/ayavisionbench/judge_templates.py
 create mode 100644 lmms_eval/tasks/ayavisionbench/utils.py

diff --git a/lmms_eval/tasks/aya-vision-bench/utils.py b/lmms_eval/tasks/aya-vision-bench/utils.py
deleted file mode 100644
index abf3f767a..000000000
--- a/lmms_eval/tasks/aya-vision-bench/utils.py
+++ /dev/null
@@ -1,81 +0,0 @@
-import base64
-from PIL import Image
-from io import BytesIO
-import numpy as np
-import re
-import json
-import os
-from loguru import logger
-
-
-def base64_to_bytes(base64_string):
-    # Remove the header if it exists (e.g., "data:image/jpeg;base64,")
-    if "base64," in base64_string:
-        base64_string = base64_string.split("base64,")[1]
-    # Decode base64 string to bytes
-    img_bytes = base64.b64decode(base64_string)
-    return img_bytes
-
-def process_docs(docs):
-    """
-    Process documents...
-    """
-    # logger.info(f"processing docs")
-    # Process images in place
-    docs = docs.select(range(5)) # filter out some samples!
-    return docs
-
-
-def gen_doc_to_visual(doc):
-    image = doc['image'][0]
-    image = image.convert('RGB')
-    return [image]
-
-
-def gen_doc_to_text(doc,lmms_eval_specific_kwargs=None ):
-    question = doc["question"]
-    if 'pre_prompt' in lmms_eval_specific_kwargs:
-        pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
-    else:
-        pre_prompt = ""
-    if 'post_prompt' in lmms_eval_specific_kwargs:
-        post_prompt = lmms_eval_specific_kwargs["post_prompt"]
-    else:
-        post_prompt = ""
-    return f"{pre_prompt}Question: {question}\n{post_prompt}"
-
-
-# def doc_to_choice(doc,lmms_eval_specific_kwargs=None ):
-#     choices = doc["choices"]
-#     choice_list = [f"{label}: {text}" for label, text in zip(choices["label"], choices["text"])]
-#     choice_list = choices["text"]
-#     return choice_list
-
-# def doc_to_target(doc,lmms_eval_specific_kwargs=None ):
-#     choices = doc["choices"]
-#     answerKey = doc["answerKey"]
-#     import pdb; pdb.set_trace()
-#     return choices["label"].index(answerKey)
-
-def gen_process_results(doc, results):
-
-    generated_text = [res[0] for res in results]
-    
-    return {"results": {
-        "id": doc["index"],
-        "image": doc["image"],
-        "question": doc["question"],
-        "image_category": doc["image_source_category"],
-        "prediction": generated_text 
-        } 
-    }
-
-
-# def save_result_to_cache(doc, round_res, previous_round_info, save_dir):
-#     save_dict = dict(
-#         sample_id=doc["index"],
-#         question=doc["question"],
-#         round_res=round_res,
-#     )
-#     save_dict.update(previous_round_info)
-#     json.dump(save_dict, open(os.path.join(save_dir, f"{save_dict['sample_id']}.json"), "w"), indent=4)
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-cs.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml
similarity index 100%
rename from lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-cs.yaml
rename to lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-de.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml
similarity index 100%
rename from lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-de.yaml
rename to lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-eng.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-eng.yaml
similarity index 87%
rename from lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-eng.yaml
rename to lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-eng.yaml
index d344b49b9..d0f3463bf 100644
--- a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-eng.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-eng.yaml
@@ -8,6 +8,7 @@ process_docs: !function utils.process_docs
 doc_to_visual: !function utils.gen_doc_to_visual
 doc_to_text: !function utils.gen_doc_to_text
 doc_to_target: ""
+process_results: !function utils.gen_process_results
 
 generation_kwargs:
   max_new_tokens: 1024
@@ -16,8 +17,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-process_results: !function utils.gen_process_results
-
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
 
 metadata:
   - version: 0.0
@@ -29,3 +31,4 @@ lmms_eval_specific_kwargs:
 
 
 
+
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-es.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml
similarity index 100%
rename from lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-es.yaml
rename to lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-fr.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml
similarity index 100%
rename from lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-fr.yaml
rename to lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-it.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml
similarity index 100%
rename from lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-it.yaml
rename to lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-ko.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml
similarity index 100%
rename from lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-ko.yaml
rename to lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-nl.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml
similarity index 100%
rename from lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-nl.yaml
rename to lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-pt.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml
similarity index 100%
rename from lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-pt.yaml
rename to lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-ru.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml
similarity index 100%
rename from lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-ru.yaml
rename to lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-zh.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml
similarity index 100%
rename from lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen-zh.yaml
rename to lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml
diff --git a/lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen.yaml
similarity index 100%
rename from lmms_eval/tasks/aya-vision-bench/aya-vision-bench-gen.yaml
rename to lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen.yaml
diff --git a/lmms_eval/tasks/ayavisionbench/eval_with_judge_template.yaml b/lmms_eval/tasks/ayavisionbench/eval_with_judge_template.yaml
new file mode 100644
index 000000000..287a3c76a
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/eval_with_judge_template.yaml
@@ -0,0 +1,13 @@
+# if comparative, then the judge will compare the baseline model output with the predicted outputs. If the baseline model outputs path is not found, then the predicted outputs will be just saved, without running the judge.
+# if direct assessment, then the judge will just judge the predicted outputs!
+# to disable the judge, set run_judge to false
+run_judge: true
+judge_model_name: gpt-4o-mini
+judge_prompt_type: comparative 
+baseline_model_outputs_path: /mnt/scratch-artemis/manos/data/tower-vision-eval-outputs/tower-vision/text-only-results/aya-vision-bench-gen/Unbabel__lnext-qwen2p5-7b-siglip2-v5/Unbabel__lnext-qwen2p5-7b-siglip2-v5/20250407_230052_samples_aya-vision-bench-gen-eng.jsonl
+max_tokens: 1024
+temperature: 0
+max_retries: 3
+wait_time: 10
+api_key: sk-proj-1234567890
+api_url: https://api.openai.com/v1/chat/completions
\ No newline at end of file
diff --git a/lmms_eval/tasks/ayavisionbench/judge_templates.py b/lmms_eval/tasks/ayavisionbench/judge_templates.py
new file mode 100755
index 000000000..84530d71f
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/judge_templates.py
@@ -0,0 +1,39 @@
+COMPARATIVE_GEN_SYSTEM_PROMPT = """\
+Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user prompt displayed below. You will be given assistant A's answer and assistant B's answer. Your job is to evaluate which assistant's answer is better.
+
+Begin your evaluation by generating your own answer to the prompt. You must provide your answers before judging any answers.
+
+When evaluating the assistants' answers, compare both assistants' answers with your answer. You must identify and correct any mistakes or inaccurate information.
+
+Then consider if the assistant's answers are helpful, relevant, and concise. Helpful means the answer correctly responds to the prompt or follows the instructions. Note when user prompt has any ambiguity or more than one interpretation, it is more helpful and appropriate to ask for clarifications or more information from the user than providing an answer based on assumptions. Relevant means all parts of the response closely connect or are appropriate to what is being asked. Concise means the response is clear and not verbose or excessive.
+
+Then consider the creativity and novelty of the assistant's answers when needed. Finally, identify any missing important information in the assistants' answers that would be beneficial to include when responding to the user prompt.
+
+After providing your explanation, you must output only one of the following choices as your final verdict with a label:
+
+1. Assistant A is significantly better: [[A>>B]]
+2. Assistant A is slightly better: [[A>B]]
+3. Tie, relatively the same: [[A=B]]
+4. Assistant B is slightly better: [[B>A]]
+5. Assistant B is significantly better: [[B>>A]]
+
+Example output: "My final verdict is tie: [[A=B]]".\
+"""
+
+COMPARATIVE_GEN_USER_PROMPT = """Question:{question}\n\n<|The Start of Assistant A's Answer|>\n{answer_1}\n<|The End of Assistant A's Answer|>\n\n<|The Start of Assistant B's Answer|>\n{answer_2}\n<|The End of Assistant B's Answer|>"""
+
+DIRECT_ASSESSMENT_SYSTEM_PROMPT = """\
+You are an impartial judge and will be given an image, a question, and an answer of an AI assistant. Your job is to evaluate the quality of the answer.
+
+Begin your evaluation by generating your own answer to the prompt. You must provide your answers before judging any answers.
+
+When evaluating the assistant's answer, compare the answer with your answer. You must identify and correct any mistakes or inaccurate information.
+
+Then consider if the assistant's answer is helpful, relevant, and concise. Helpful means the answer correctly responds to the prompt or follows the instructions. Note when user prompt has any ambiguity or more than one interpretation, it is more helpful and appropriate to ask for clarifications or more information from the user than providing an answer based on assumptions. Relevant means all parts of the response closely connect or are appropriate to what is being asked. Concise means the response is clear and not verbose or excessive.
+
+After providing your explanation, you must output only a score between 0 and 100, where 0 is the worst and 100 is the best.
+
+Example output: "My final score is 85."
+"""
+
+DIRECT_ASSESSMENT_USER_PROMPT = """Question:{question}\n\n<|The Start of Assistant Answer|>\n{answer}\n<|The End of Assistant Answer|>\n\n"""
\ No newline at end of file
diff --git a/lmms_eval/tasks/ayavisionbench/utils.py b/lmms_eval/tasks/ayavisionbench/utils.py
new file mode 100644
index 000000000..ceaf65dfe
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/utils.py
@@ -0,0 +1,231 @@
+import base64
+from PIL import Image
+from io import BytesIO
+import numpy as np
+import re
+import json
+import os
+from loguru import logger
+from pathlib import Path
+import yaml
+
+from lmms_eval.tasks.ayavisionbench.judge_templates import (
+    COMPARATIVE_GEN_USER_PROMPT,
+    COMPARATIVE_GEN_SYSTEM_PROMPT,
+    DIRECT_ASSESSMENT_SYSTEM_PROMPT,
+    DIRECT_ASSESSMENT_USER_PROMPT 
+)
+import requests
+import time
+
+def get_judge_config():
+    with open(Path(__file__).parent / "eval_with_judge_template.yaml", "r") as f:
+        raw_data = f.readlines()
+        safe_data = []
+        for i, line in enumerate(raw_data):
+            # remove function definition since yaml load cannot handle it
+            if "!function" not in line:
+                safe_data.append(line)
+        config = yaml.safe_load("".join(safe_data))
+    return config
+
+
+def run_judge(questions,preds,judge_config,baseline_model_outputs=None,images=None):
+    logger.info(f"Selected judge type: {judge_config['judge_prompt_type']}")
+    api_key = judge_config["api_key"]
+    api_url = judge_config["api_url"]
+    judge_model_name = judge_config["judge_model_name"]
+    headers = {
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+        }
+    payload = {
+        "model": judge_model_name,
+        "max_tokens": judge_config["max_tokens"],
+        "temperature": judge_config["temperature"],
+    }
+
+    if judge_config["judge_prompt_type"] == "comparative_gen":
+        system_prompt = COMPARATIVE_GEN_SYSTEM_PROMPT
+        user_prompt_template = COMPARATIVE_GEN_USER_PROMPT
+        prompts = [user_prompt_template.format(question=question,answer_1=base_output,answer_2=pred) for question,pred,base_output in zip(questions,preds,baseline_model_outputs)]
+    elif judge_config["judge_prompt_type"] == "direct_assessment":
+        system_prompt = DIRECT_ASSESSMENT_SYSTEM_PROMPT
+        user_prompt_template = DIRECT_ASSESSMENT_USER_PROMPT
+        prompts = [user_prompt_template.format(question=question,answer=pred) for question,pred in zip(questions,preds)]
+    else:
+        raise ValueError(f"Invalid judge prompt type: {judge_config['judge_prompt_type']}")
+    
+    messages = []
+    for image,prompt in zip(images,prompts):
+        if image is not None:
+            messages.append([
+                    {"role": "system", "content": [{"type": "text", "text": system_prompt}]},
+                    {"role": "user",
+                    "content": [{"type": "text", "text": prompt},{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64, {image}"}},],
+                    },])
+        else:
+            messages.append([
+                    {"role": "system", "content": [{"type": "text", "text": system_prompt}]},
+                    {"role": "user", "content": [{"type": "text", "text": prompt}]},
+                ])
+    payload["messages"] = messages
+
+    responses = []
+    for attempt in range(judge_config["max_retries"]):
+        try:
+            response = requests.post(api_url, headers=headers, json=payload, timeout=60)
+            response.raise_for_status()
+            response_data = response.json()
+            # print(response_data)
+            responses.append(response_data)
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Request failed on attempt {attempt+1}: {e}")
+            time.sleep(judge_config["wait_time"])
+            if attempt == judge_config["max_retries"] - 1:
+                logger.info(f"Failed to get response after {judge_config['max_retries']} attempts")
+                responses.append(None)
+        except Exception as e:
+            logger.info(f"Error on attempt {attempt+1}: {e}")
+            time.sleep(judge_config["wait_time"])
+            responses.append(None)
+    return response_data
+
+
+def process_judge_results(responses):
+    import pdb; pdb.set_trace()
+    for response in responses:
+        if response is None:
+            continue
+        response_data = response["choices"][0]["message"]["content"]
+        responses.append(response_data)
+    return responses
+
+def load_baselined_outputs(baseline_model_outputs_path):
+    filtered_responses = []
+    with open(baseline_model_outputs_path, 'r', encoding='utf-8') as f:
+        for line in f:
+            try:
+                data = json.loads(line.strip())
+                if 'filtered_resps' in data:
+                    # Handle both single string and list of strings cases
+                    resps = data['filtered_resps']
+                    if isinstance(resps, list):
+                        filtered_responses.extend(resps)
+                    else:
+                        filtered_responses.append(resps)
+            except json.JSONDecodeError:
+                print(f"Warning: Skipping invalid JSON line")
+                continue
+    return filtered_responses
+
+
+def base64_to_bytes(base64_string):
+    # Remove the header if it exists (e.g., "data:image/jpeg;base64,")
+    if "base64," in base64_string:
+        base64_string = base64_string.split("base64,")[1]
+    # Decode base64 string to bytes
+    img_bytes = base64.b64decode(base64_string)
+    return img_bytes
+
+def process_docs(docs):
+    """
+    Process documents...
+    """
+    # logger.info(f"processing docs")
+    # Process images in place
+    docs = docs.select(range(5)) # filter out some samples!
+    return docs
+
+
+def gen_doc_to_visual(doc):
+    image = doc['image'][0]
+    image = image.convert('RGB')
+    return [image]
+
+
+def gen_doc_to_text(doc,lmms_eval_specific_kwargs=None ):
+    question = doc["question"]
+    if 'pre_prompt' in lmms_eval_specific_kwargs:
+        pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    else:
+        pre_prompt = ""
+    if 'post_prompt' in lmms_eval_specific_kwargs:
+        post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+    else:
+        post_prompt = ""
+    return f"{pre_prompt}Question: {question}\n{post_prompt}"
+
+
+# def doc_to_choice(doc,lmms_eval_specific_kwargs=None ):
+#     choices = doc["choices"]
+#     choice_list = [f"{label}: {text}" for label, text in zip(choices["label"], choices["text"])]
+#     choice_list = choices["text"]
+#     return choice_list
+
+# def doc_to_target(doc,lmms_eval_specific_kwargs=None ):
+#     choices = doc["choices"]
+#     answerKey = doc["answerKey"]
+#     import pdb; pdb.set_trace()
+#     return choices["label"].index(answerKey)
+
+def gen_process_results(doc, results):
+    generated_texts = [res[0] for res in results]
+    return {"results": {
+        "id": doc["index"],
+        "image": doc["image"],
+        "question": doc["question"],
+        "image_category": doc["image_source_category"],
+        "prediction": generated_texts
+        } 
+    }
+
+def aggregate_results(results):
+    preds = [result["prediction"] for result in results]
+    questions = [result["question"] for result in results]
+    images = [result["image"] for result in results]
+
+    judge_config = get_judge_config()
+    logger.info("Checking judge config...")
+    if judge_config["run_judge"]:
+        logger.info("Judge is enabled, checking judge config...")
+        if judge_config["judge_prompt_type"] == "comparative":
+            logger.info("Judgement is set to comparative, checking baseline model outputs...")
+            try:
+                baseline_model_outputs = load_baselined_outputs(judge_config["baseline_model_outputs_path"])
+                assert len(baseline_model_outputs) == len(preds)
+            except Exception as e:
+                logger.error(f"Failed loading baseline model outputs.")
+                raise e
+            
+            logger.info("Baseline model outputs loaded successfully.")
+            #TODO: run judge with comparative prompt
+            judge_results = run_judge(questions,preds,judge_config,baseline_model_outputs,images)
+
+        elif judge_config["judge_prompt_type"] == "direct assessment":
+            preds = [result["prediction"] for result in results]
+            #Note: run judge on predicted outputs
+            judge_results = run_judge(questions,preds,judge_config,baseline_model_outputs=None,images=images)
+        else:
+            logger.error(f"Invalid judge prompt type: {judge_config['judge_prompt_type']}.Skipping judge...")
+            judge_results = None
+        
+    else:
+        judge_results = None
+        logger.info("Judge is disabled, skipping judge...")
+
+    if judge_results is None:
+        return {"judge_results": None}
+    else:
+        results = process_judge_results(judge_results)
+        return results
+
+
+# def cache_judge_outputs(doc, round_res, previous_round_info, save_dir):
+#     save_dict = dict(
+#         sample_id=doc["index"],
+#         question=doc["question"],
+#         round_res=round_res,
+#     )
+#     save_dict.update(previous_round_info)
+#     json.dump(save_dict, open(os.path.join(save_dir, f"{save_dict['sample_id']}.json"), "w"), indent=4)

From 2665a733a1a6d79a3ef13a589ee23396473f9a3c Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Tue, 15 Apr 2025 13:16:03 +0000
Subject: [PATCH 081/112] updated aya vision bench

---
 .../eval_with_judge_template.yaml             |  10 +-
 lmms_eval/tasks/ayavisionbench/utils.py       | 138 +++++++++++++-----
 2 files changed, 110 insertions(+), 38 deletions(-)

diff --git a/lmms_eval/tasks/ayavisionbench/eval_with_judge_template.yaml b/lmms_eval/tasks/ayavisionbench/eval_with_judge_template.yaml
index 287a3c76a..f822fcbdb 100644
--- a/lmms_eval/tasks/ayavisionbench/eval_with_judge_template.yaml
+++ b/lmms_eval/tasks/ayavisionbench/eval_with_judge_template.yaml
@@ -2,12 +2,12 @@
 # if direct assessment, then the judge will just judge the predicted outputs!
 # to disable the judge, set run_judge to false
 run_judge: true
-judge_model_name: gpt-4o-mini
-judge_prompt_type: comparative 
-baseline_model_outputs_path: /mnt/scratch-artemis/manos/data/tower-vision-eval-outputs/tower-vision/text-only-results/aya-vision-bench-gen/Unbabel__lnext-qwen2p5-7b-siglip2-v5/Unbabel__lnext-qwen2p5-7b-siglip2-v5/20250407_230052_samples_aya-vision-bench-gen-eng.jsonl
+judge_model_name: o1-mini-2024-09-12
+judge_prompt_type: comparative
+text_only: true
+baseline_model_outputs_path: /mnt/scratch-artemis/manos/data/tower-vision-eval-outputs/tower-vision/text-only-results-debug/aya-vision-bench-gen-eng/Unbabel__lnext-qwen2p5-7b-siglip2-v5/Unbabel__lnext-qwen2p5-7b-siglip2-v5/20250415_044006_samples_aya-vision-bench-gen-eng.jsonl
 max_tokens: 1024
 temperature: 0
 max_retries: 3
 wait_time: 10
-api_key: sk-proj-1234567890
-api_url: https://api.openai.com/v1/chat/completions
\ No newline at end of file
+api_url: https://api.openai.com/v1/chat/completions  #https://api.anthropic.com/v1/messages  for anthropic
\ No newline at end of file
diff --git a/lmms_eval/tasks/ayavisionbench/utils.py b/lmms_eval/tasks/ayavisionbench/utils.py
index ceaf65dfe..321b38cbd 100644
--- a/lmms_eval/tasks/ayavisionbench/utils.py
+++ b/lmms_eval/tasks/ayavisionbench/utils.py
@@ -8,6 +8,11 @@
 from loguru import logger
 from pathlib import Path
 import yaml
+import io
+
+from dotenv import load_dotenv
+
+
 
 from lmms_eval.tasks.ayavisionbench.judge_templates import (
     COMPARATIVE_GEN_USER_PROMPT,
@@ -32,20 +37,37 @@ def get_judge_config():
 
 def run_judge(questions,preds,judge_config,baseline_model_outputs=None,images=None):
     logger.info(f"Selected judge type: {judge_config['judge_prompt_type']}")
-    api_key = judge_config["api_key"]
+    # # Load environment variables from .env file
+    load_dotenv()
+    
     api_url = judge_config["api_url"]
+    # api_key = judge_config["api_key"]
     judge_model_name = judge_config["judge_model_name"]
-    headers = {
+
+    if 'anthropic' in api_url:
+        api_key = os.getenv('ANTHROPIC_API_KEY')
+        headers = {
+        "x-api-key": api_key,  # Different header for Claude
+        "Content-Type": "application/json",
+        "anthropic-version": "2023-06-01"  # Required for Claude
+    }
+    elif 'openai' in api_url:
+        api_key = os.getenv('OPENAI_API_KEY')
+        headers = {
             "Authorization": f"Bearer {api_key}",
             "Content-Type": "application/json",
         }
+    else:
+        logger.error(f"Invalid Api URL and API key combination")
+        raise ValueError(f"Invalid API URL: {api_url}")
+
     payload = {
         "model": judge_model_name,
         "max_tokens": judge_config["max_tokens"],
         "temperature": judge_config["temperature"],
     }
 
-    if judge_config["judge_prompt_type"] == "comparative_gen":
+    if judge_config["judge_prompt_type"] == "comparative":
         system_prompt = COMPARATIVE_GEN_SYSTEM_PROMPT
         user_prompt_template = COMPARATIVE_GEN_USER_PROMPT
         prompts = [user_prompt_template.format(question=question,answer_1=base_output,answer_2=pred) for question,pred,base_output in zip(questions,preds,baseline_model_outputs)]
@@ -58,7 +80,8 @@ def run_judge(questions,preds,judge_config,baseline_model_outputs=None,images=No
     
     messages = []
     for image,prompt in zip(images,prompts):
-        if image is not None:
+        #TODO: change image format to base64
+        if image is not None or judge_config["text_only"]==False:
             messages.append([
                     {"role": "system", "content": [{"type": "text", "text": system_prompt}]},
                     {"role": "user",
@@ -66,31 +89,70 @@ def run_judge(questions,preds,judge_config,baseline_model_outputs=None,images=No
                     },])
         else:
             messages.append([
-                    {"role": "system", "content": [{"type": "text", "text": system_prompt}]},
-                    {"role": "user", "content": [{"type": "text", "text": prompt}]},
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": prompt}
                 ])
     payload["messages"] = messages
 
     responses = []
-    for attempt in range(judge_config["max_retries"]):
-        try:
-            response = requests.post(api_url, headers=headers, json=payload, timeout=60)
-            response.raise_for_status()
-            response_data = response.json()
-            # print(response_data)
-            responses.append(response_data)
-        except requests.exceptions.RequestException as e:
-            logger.error(f"Request failed on attempt {attempt+1}: {e}")
-            time.sleep(judge_config["wait_time"])
-            if attempt == judge_config["max_retries"] - 1:
-                logger.info(f"Failed to get response after {judge_config['max_retries']} attempts")
-                responses.append(None)
-        except Exception as e:
-            logger.info(f"Error on attempt {attempt+1}: {e}")
-            time.sleep(judge_config["wait_time"])
-            responses.append(None)
-    return response_data
 
+    # for attempt in range(judge_config["max_retries"]):
+    #     try:
+    #         response = requests.post(api_url, headers=headers, json=payload, timeout=60)
+    #         response.raise_for_status()
+    #         import pdb; pdb.set_trace()
+    #         response_data = response.json()
+    #         # print(response_data)
+    #         responses.append(response_data)
+    #         break
+    #     except requests.exceptions.RequestException as e:
+    #         logger.error(f"Request failed on attempt {attempt+1}: {e}")
+    #         time.sleep(judge_config["wait_time"])
+    #         if attempt == judge_config["max_retries"] - 1:
+    #             logger.info(f"Failed to get response after {judge_config['max_retries']} attempts")
+    #             responses.append(None)
+    #     except Exception as e:
+    #         logger.info(f"Error on attempt {attempt+1}: {e}")
+    #         time.sleep(judge_config["wait_time"])
+    #         responses.append(None)
+
+    #Do the api call
+    response = api_call(api_url,headers,payload)
+    return response
+
+def api_call(api_url,headers,payload):
+    try:
+        response = requests.post(api_url, headers=headers, json=payload, timeout=60)
+        response.raise_for_status()  # Raises an HTTPError for bad responses (4xx, 5xx)
+        response_data = response.json()
+        return response_data
+        
+    except requests.exceptions.HTTPError as e:
+        logger.error(f"HTTP error occurred: {e}")
+        logger.error(f"Response status code: {e.response.status_code}")
+        logger.error(f"Response text: {e.response.text}")
+        return None
+        
+    except requests.exceptions.ConnectionError as e:
+        logger.error(f"Error connecting to the server: {e}")
+        return None
+        
+    except requests.exceptions.Timeout as e:
+        logger.error(f"Request timed out: {e}")
+        return None
+        
+    except requests.exceptions.RequestException as e:
+        logger.error(f"An error occurred while making the request: {e}")
+        return None
+        
+    except json.JSONDecodeError as e:
+        logger.error(f"Error decoding JSON response: {e}")
+        logger.error(f"Response text: {response.text}")
+        return None
+        
+    except Exception as e:
+        logger.error(f"An unexpected error occurred: {e}")
+        return None
 
 def process_judge_results(responses):
     import pdb; pdb.set_trace()
@@ -101,7 +163,7 @@ def process_judge_results(responses):
         responses.append(response_data)
     return responses
 
-def load_baselined_outputs(baseline_model_outputs_path):
+def load_baseline_outputs(baseline_model_outputs_path):
     filtered_responses = []
     with open(baseline_model_outputs_path, 'r', encoding='utf-8') as f:
         for line in f:
@@ -135,6 +197,12 @@ def process_docs(docs):
     # logger.info(f"processing docs")
     # Process images in place
     docs = docs.select(range(5)) # filter out some samples!
+    # Create proper copies of images
+    def copy_image_fn(example):
+        example['copy_image'] = example['image'].copy()
+        return example
+    
+    docs = docs.map(copy_image_fn)
     return docs
 
 
@@ -170,10 +238,13 @@ def gen_doc_to_text(doc,lmms_eval_specific_kwargs=None ):
 #     return choices["label"].index(answerKey)
 
 def gen_process_results(doc, results):
-    generated_texts = [res[0] for res in results]
+    generated_texts = results[0]
+    if isinstance(doc['copy_image'][0]['bytes'], bytes):
+        img = Image.open(io.BytesIO(doc['copy_image'][0]['bytes'])).convert('RGB')
+        myimg = img
     return {"results": {
         "id": doc["index"],
-        "image": doc["image"],
+        "image": myimg,
         "question": doc["question"],
         "image_category": doc["image_source_category"],
         "prediction": generated_texts
@@ -186,22 +257,23 @@ def aggregate_results(results):
     images = [result["image"] for result in results]
 
     judge_config = get_judge_config()
+
     logger.info("Checking judge config...")
     if judge_config["run_judge"]:
         logger.info("Judge is enabled, checking judge config...")
         if judge_config["judge_prompt_type"] == "comparative":
             logger.info("Judgement is set to comparative, checking baseline model outputs...")
             try:
-                baseline_model_outputs = load_baselined_outputs(judge_config["baseline_model_outputs_path"])
+                baseline_model_outputs = load_baseline_outputs(judge_config["baseline_model_outputs_path"])
                 assert len(baseline_model_outputs) == len(preds)
+                logger.info("Baseline model outputs loaded successfully.")
+                #TODO: run judge with comparative prompt
+                import pdb; pdb.set_trace()
+                judge_results = run_judge(questions,preds,judge_config,baseline_model_outputs,images)
             except Exception as e:
-                logger.error(f"Failed loading baseline model outputs.")
+                logger.error(f"Failed while loading model outputs or running judge.")
                 raise e
             
-            logger.info("Baseline model outputs loaded successfully.")
-            #TODO: run judge with comparative prompt
-            judge_results = run_judge(questions,preds,judge_config,baseline_model_outputs,images)
-
         elif judge_config["judge_prompt_type"] == "direct assessment":
             preds = [result["prediction"] for result in results]
             #Note: run judge on predicted outputs

From 1d3bd71239c96f3f620b065184d39e5cd585691b Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Tue, 15 Apr 2025 22:10:12 +0000
Subject: [PATCH 082/112] aya vision bench updated + added comparative judge

---
 .../eval_with_judge_template.yaml             |   9 +-
 .../tasks/ayavisionbench/judge_templates.py   |   4 +-
 lmms_eval/tasks/ayavisionbench/judge_utils.py | 272 ++++++++++++++++++
 lmms_eval/tasks/ayavisionbench/utils.py       | 221 ++------------
 4 files changed, 302 insertions(+), 204 deletions(-)
 create mode 100644 lmms_eval/tasks/ayavisionbench/judge_utils.py

diff --git a/lmms_eval/tasks/ayavisionbench/eval_with_judge_template.yaml b/lmms_eval/tasks/ayavisionbench/eval_with_judge_template.yaml
index f822fcbdb..439aaa35c 100644
--- a/lmms_eval/tasks/ayavisionbench/eval_with_judge_template.yaml
+++ b/lmms_eval/tasks/ayavisionbench/eval_with_judge_template.yaml
@@ -2,12 +2,11 @@
 # if direct assessment, then the judge will just judge the predicted outputs!
 # to disable the judge, set run_judge to false
 run_judge: true
-judge_model_name: o1-mini-2024-09-12
+api_type: openai #openai #anthropic
+judge_model_name: gpt-4o-mini #gpt-4o-mini #claude-3-7-sonnet-20250219
 judge_prompt_type: comparative
 text_only: true
-baseline_model_outputs_path: /mnt/scratch-artemis/manos/data/tower-vision-eval-outputs/tower-vision/text-only-results-debug/aya-vision-bench-gen-eng/Unbabel__lnext-qwen2p5-7b-siglip2-v5/Unbabel__lnext-qwen2p5-7b-siglip2-v5/20250415_044006_samples_aya-vision-bench-gen-eng.jsonl
+baseline_model_outputs_path: /mnt/scratch-artemis/manos/data/tower-vision-eval-outputs/tower-vision/tower-vision-debug-results/aya-vision-bench-gen-eng/Unbabel__lnext-qwen2p5-7b-siglip2-v5/Unbabel__lnext-qwen2p5-7b-siglip2-v5/20250416_060306_samples_aya-vision-bench-gen-eng.jsonl
 max_tokens: 1024
 temperature: 0
-max_retries: 3
-wait_time: 10
-api_url: https://api.openai.com/v1/chat/completions  #https://api.anthropic.com/v1/messages  for anthropic
\ No newline at end of file
+api_url: https://api.openai.com/v1/chat/completions    #https://api.anthropic.com/v1/messages  for anthropic
\ No newline at end of file
diff --git a/lmms_eval/tasks/ayavisionbench/judge_templates.py b/lmms_eval/tasks/ayavisionbench/judge_templates.py
index 84530d71f..6a635ed65 100755
--- a/lmms_eval/tasks/ayavisionbench/judge_templates.py
+++ b/lmms_eval/tasks/ayavisionbench/judge_templates.py
@@ -1,7 +1,7 @@
 COMPARATIVE_GEN_SYSTEM_PROMPT = """\
-Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user prompt displayed below. You will be given assistant A's answer and assistant B's answer. Your job is to evaluate which assistant's answer is better.
+Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user prompt displayed below. You will be given assistant A's answer and assistant B's answer. Your job is to evaluate which assistant's answer is better, by taking into account the given image.
 
-Begin your evaluation by generating your own answer to the prompt. You must provide your answers before judging any answers.
+Begin your evaluation by generating your own answer to the prompt. You must provide your answers before judging any answers. To provide your answer, take into account the given image.
 
 When evaluating the assistants' answers, compare both assistants' answers with your answer. You must identify and correct any mistakes or inaccurate information.
 
diff --git a/lmms_eval/tasks/ayavisionbench/judge_utils.py b/lmms_eval/tasks/ayavisionbench/judge_utils.py
new file mode 100644
index 000000000..bcdbd8c5d
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/judge_utils.py
@@ -0,0 +1,272 @@
+import yaml
+from pathlib import Path
+import os
+import requests
+import json
+import re
+from loguru import logger
+from dotenv import load_dotenv
+import base64
+import io
+from PIL import Image
+from tqdm import tqdm
+from typing import Dict, List, Optional, Union
+
+from lmms_eval.tasks.ayavisionbench.judge_templates import (
+    COMPARATIVE_GEN_USER_PROMPT,
+    COMPARATIVE_GEN_SYSTEM_PROMPT,
+    DIRECT_ASSESSMENT_SYSTEM_PROMPT,
+    DIRECT_ASSESSMENT_USER_PROMPT 
+)
+
+
+def img_bytes_to_url(image_dict: dict) -> str:
+    """
+    Convert image bytes to base64 URL.
+    
+    Args:
+        image_dict: Dictionary containing image bytes
+        
+    Returns:
+        str: base64 URL-encoded JPEG string
+    """
+    try:
+        # Convert bytes to PIL Image
+        image = Image.open(io.BytesIO(image_dict['bytes']))
+        
+        # Convert to RGB if needed
+        if image.mode != 'RGB':
+            image = image.convert('RGB')
+        
+        # Save as JPEG to buffer
+        buffer = io.BytesIO()
+        image.save(buffer, format="JPEG")
+        img_bytes = buffer.getvalue()
+        
+        # Convert to base64
+        base64_image = base64.b64encode(img_bytes).decode('utf-8')
+        
+        # Create data URL
+        data_url = f"data:image/jpeg;base64,{base64_image}"
+        
+        return data_url
+        
+    except Exception as e:
+        logger.error(f"Error converting image bytes to URL: {str(e)}")
+        return None
+
+def get_judge_config():
+    with open(Path(__file__).parent / "eval_with_judge_template.yaml", "r") as f:
+        raw_data = f.readlines()
+        safe_data = []
+        for i, line in enumerate(raw_data):
+            # remove function definition since yaml load cannot handle it
+            if "!function" not in line:
+                safe_data.append(line)
+        config = yaml.safe_load("".join(safe_data))
+    return config
+
+
+def choose_api(api_url,judge_config):
+    judge_model_name = judge_config["judge_model_name"]
+    if judge_config["api_type"] == "anthropic":
+        api_key = os.getenv('ANTHROPIC_API_KEY')
+        headers = {
+        "x-api-key": api_key,  # Different header for Claude
+        "Content-Type": "application/json",
+        "anthropic-version": "2023-06-01"  # Required for Claude
+    }
+    elif judge_config["api_type"] == "openai":
+        api_key = os.getenv('OPENAI_API_KEY')
+        headers = {
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+        }
+    else:
+        logger.error(f"API type not supported. Please implement it!")
+        raise ValueError(f"Invalid API type: {judge_config['api_type']}")
+
+    payload = {
+        "model": judge_model_name,
+        "max_tokens": judge_config["max_tokens"],
+        "temperature": judge_config["temperature"],
+    }
+    return headers,payload
+
+
+def api_call(api_url,headers,payload):
+    try:
+        response = requests.post(api_url, headers=headers, json=payload, timeout=60)
+        response.raise_for_status()  # Raises an HTTPError for bad responses (4xx, 5xx)
+        response_data = response.json()
+        return response_data
+        
+    except requests.exceptions.HTTPError as e:
+        logger.error(f"HTTP error occurred: {e}")
+        logger.error(f"Response status code: {e.response.status_code}")
+        logger.error(f"Response text: {e.response.text}")
+        return None
+        
+    except requests.exceptions.ConnectionError as e:
+        logger.error(f"Error connecting to the server: {e}")
+        return None
+        
+    except requests.exceptions.Timeout as e:
+        logger.error(f"Request timed out: {e}")
+        return None
+        
+    except requests.exceptions.RequestException as e:
+        logger.error(f"An error occurred while making the request: {e}")
+        return None
+        
+    except json.JSONDecodeError as e:
+        logger.error(f"Error decoding JSON response: {e}")
+        logger.error(f"Response text: {response.text}")
+        return None
+        
+    except Exception as e:
+        logger.error(f"An unexpected error occurred: {e}")
+        return None
+
+
+def set_prompts(judge_config,questions,preds,baseline_model_outputs=None):
+    if judge_config["judge_prompt_type"] == "comparative":
+        system_prompt = COMPARATIVE_GEN_SYSTEM_PROMPT
+        user_prompt_template = COMPARATIVE_GEN_USER_PROMPT
+        prompts = [user_prompt_template.format(question=question,answer_1=base_output,answer_2=pred) for question,pred,base_output in zip(questions,preds,baseline_model_outputs)]
+    elif judge_config["judge_prompt_type"] == "direct_assessment":
+        system_prompt = DIRECT_ASSESSMENT_SYSTEM_PROMPT
+        user_prompt_template = DIRECT_ASSESSMENT_USER_PROMPT
+        prompts = [user_prompt_template.format(question=question,answer=pred) for question,pred in zip(questions,preds)]
+    else:
+        raise ValueError(f"Invalid judge prompt type: {judge_config['judge_prompt_type']}")
+    return system_prompt, prompts
+
+
+def run_judge(
+    questions: List[str],
+    preds: List[str],
+    judge_config: Dict,
+    baseline_model_outputs: Optional[List[str]] = None,
+    images: Optional[Dict[str, Union[bytes, None]]] = None
+) -> List[str]:
+    """
+    Run judge evaluation on predictions with optional image inputs.
+    
+    Args:
+        questions: List of questions to evaluate
+        preds: List of predicted outputs to evaluate
+        judge_config: Configuration dictionary for the judge
+        baseline_model_outputs: Optional list of baseline outputs to compare against
+        images: Optional dictionary containing image data with format {'bytes': bytes, 'path': None}
+        
+    Returns:
+        List[Dict]: List of parsed responses from the judge
+    """
+    logger.info(f"Selected judge type: {judge_config['judge_prompt_type']}")
+    # # Load environment variables from .env file
+    load_dotenv()
+    
+    api_url = judge_config["api_url"]
+
+    headers,payload = choose_api(api_url,judge_config)
+    system_prompt, user_prompts = set_prompts(judge_config,questions,preds,baseline_model_outputs)
+    
+    responses = []
+    total_items = len(user_prompts)
+    logger.info(f"Running judge evaluation for {total_items} items...")
+    # Create progress bar with additional information
+    with tqdm(total=total_items, 
+              desc="Running judge evaluation", 
+              bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]',
+              ncols=100) as pbar:
+              
+        for idx, (image, prompt) in enumerate(zip(images, user_prompts)):
+            if image is not None or judge_config["text_only"]==False:
+                message=([
+                        {"role": "system", "content": [{"type": "text", "text": system_prompt}]},
+                        {"role": "user", "content": [{"type": "text", "text": prompt},{"type": "image_url", "image_url": {"url":img_bytes_to_url(image)}}]}
+                        ])
+            else:
+                message=[{"role": "system", "content": system_prompt},{"role": "user", "content": prompt}]
+            
+            payload["messages"] = message
+
+            response = api_call(api_url, headers, payload)
+            responses.append(response)
+            
+            # Update progress bar with current item
+            pbar.set_postfix({"Current": f"{idx+1}/{total_items}"})
+            pbar.update(1)
+    
+    responses = parse_judge_responses(responses,judge_config)
+    return responses
+
+
+def parse_comparative_response(response: str) -> str:
+    """
+    Extract verdict pattern from judge response.
+    Matches: [[A>>B]], [[A>B]], [[A=B]], [[B>A]], [[B>>A]]
+    
+    Args:
+        response: String containing judge's response
+        
+    Returns:
+        str: Matched verdict pattern or None if no match found
+    """
+    # Regex pattern to match all possible verdicts
+    pattern = r'\[\[(A>>B|A>B|A=B|B>A|B>>A)\]\]'
+    # Search for pattern in response
+    match = re.search(pattern, response)
+    # Return matched pattern or None
+    return match.group(0) if match else None
+
+
+def parse_judge_responses(responses,judge_config):
+    logger.info(f"Parsing responses for {judge_config['judge_prompt_type']} judge...")
+    parsed_responses = []
+    for response in responses:
+        if response is None:
+            parsed_responses.append(None)
+            continue
+        response_data = response["choices"][0]["message"]["content"]
+        if judge_config["judge_prompt_type"] == "comparative":
+            parsed_responses.append(parse_comparative_response(response_data))
+        elif judge_config["judge_prompt_type"] == "direct_assessment":
+            raise NotImplementedError("Direct assessment parsing not implemented yet!")
+    return parsed_responses
+
+
+def compute_results(responses,judge_config):
+    logger.info(f"Computing results for {judge_config['judge_prompt_type']} judge...")
+    if judge_config["judge_prompt_type"] == "comparative":
+        a_better_than_b = 0
+        a_significantly_better_than_b = 0
+        b_better_than_a = 0
+        b_significantly_better_than_a = 0
+        a_equal_to_b = 0
+        no_answer = 0
+        for response in responses:
+            if response == "[[A>>B]]":
+                a_significantly_better_than_b += 1  
+            elif response == "[[A>B]]":
+                a_better_than_b += 1
+            elif response == "[[B>>A]]":
+                b_significantly_better_than_a += 1
+            elif response == "[[B>A]]":
+                b_better_than_a += 1
+            elif response == "[[A=B]]":
+                a_equal_to_b += 1
+            else:
+                no_answer += 1
+        results = {
+            "a_better_than_b": a_better_than_b/len(responses),
+            "a_significantly_better_than_b": a_significantly_better_than_b/len(responses),
+            "b_better_than_a": b_better_than_a/len(responses),
+            "b_significantly_better_than_a": b_significantly_better_than_a/len(responses),
+            "a_equal_to_b": a_equal_to_b/len(responses),
+            "no_answer_matched": no_answer/len(responses)
+        }
+    elif judge_config["judge_prompt_type"] == "direct_assessment":
+        raise NotImplementedError("Direct assessment results computation not implemented yet!")
+    return results
\ No newline at end of file
diff --git a/lmms_eval/tasks/ayavisionbench/utils.py b/lmms_eval/tasks/ayavisionbench/utils.py
index 321b38cbd..bff95c4fa 100644
--- a/lmms_eval/tasks/ayavisionbench/utils.py
+++ b/lmms_eval/tasks/ayavisionbench/utils.py
@@ -1,167 +1,28 @@
-import base64
 from PIL import Image
 from io import BytesIO
 import numpy as np
-import re
 import json
-import os
 from loguru import logger
 from pathlib import Path
-import yaml
 import io
+from copy import deepcopy
 
-from dotenv import load_dotenv
-
-
-
-from lmms_eval.tasks.ayavisionbench.judge_templates import (
-    COMPARATIVE_GEN_USER_PROMPT,
-    COMPARATIVE_GEN_SYSTEM_PROMPT,
-    DIRECT_ASSESSMENT_SYSTEM_PROMPT,
-    DIRECT_ASSESSMENT_USER_PROMPT 
+from lmms_eval.tasks.ayavisionbench.judge_utils import (
+    get_judge_config,
+    run_judge,
+    compute_results
 )
-import requests
-import time
-
-def get_judge_config():
-    with open(Path(__file__).parent / "eval_with_judge_template.yaml", "r") as f:
-        raw_data = f.readlines()
-        safe_data = []
-        for i, line in enumerate(raw_data):
-            # remove function definition since yaml load cannot handle it
-            if "!function" not in line:
-                safe_data.append(line)
-        config = yaml.safe_load("".join(safe_data))
-    return config
-
-
-def run_judge(questions,preds,judge_config,baseline_model_outputs=None,images=None):
-    logger.info(f"Selected judge type: {judge_config['judge_prompt_type']}")
-    # # Load environment variables from .env file
-    load_dotenv()
-    
-    api_url = judge_config["api_url"]
-    # api_key = judge_config["api_key"]
-    judge_model_name = judge_config["judge_model_name"]
-
-    if 'anthropic' in api_url:
-        api_key = os.getenv('ANTHROPIC_API_KEY')
-        headers = {
-        "x-api-key": api_key,  # Different header for Claude
-        "Content-Type": "application/json",
-        "anthropic-version": "2023-06-01"  # Required for Claude
-    }
-    elif 'openai' in api_url:
-        api_key = os.getenv('OPENAI_API_KEY')
-        headers = {
-            "Authorization": f"Bearer {api_key}",
-            "Content-Type": "application/json",
-        }
-    else:
-        logger.error(f"Invalid Api URL and API key combination")
-        raise ValueError(f"Invalid API URL: {api_url}")
-
-    payload = {
-        "model": judge_model_name,
-        "max_tokens": judge_config["max_tokens"],
-        "temperature": judge_config["temperature"],
-    }
-
-    if judge_config["judge_prompt_type"] == "comparative":
-        system_prompt = COMPARATIVE_GEN_SYSTEM_PROMPT
-        user_prompt_template = COMPARATIVE_GEN_USER_PROMPT
-        prompts = [user_prompt_template.format(question=question,answer_1=base_output,answer_2=pred) for question,pred,base_output in zip(questions,preds,baseline_model_outputs)]
-    elif judge_config["judge_prompt_type"] == "direct_assessment":
-        system_prompt = DIRECT_ASSESSMENT_SYSTEM_PROMPT
-        user_prompt_template = DIRECT_ASSESSMENT_USER_PROMPT
-        prompts = [user_prompt_template.format(question=question,answer=pred) for question,pred in zip(questions,preds)]
-    else:
-        raise ValueError(f"Invalid judge prompt type: {judge_config['judge_prompt_type']}")
-    
-    messages = []
-    for image,prompt in zip(images,prompts):
-        #TODO: change image format to base64
-        if image is not None or judge_config["text_only"]==False:
-            messages.append([
-                    {"role": "system", "content": [{"type": "text", "text": system_prompt}]},
-                    {"role": "user",
-                    "content": [{"type": "text", "text": prompt},{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64, {image}"}},],
-                    },])
-        else:
-            messages.append([
-                    {"role": "system", "content": system_prompt},
-                    {"role": "user", "content": prompt}
-                ])
-    payload["messages"] = messages
-
-    responses = []
 
-    # for attempt in range(judge_config["max_retries"]):
-    #     try:
-    #         response = requests.post(api_url, headers=headers, json=payload, timeout=60)
-    #         response.raise_for_status()
-    #         import pdb; pdb.set_trace()
-    #         response_data = response.json()
-    #         # print(response_data)
-    #         responses.append(response_data)
-    #         break
-    #     except requests.exceptions.RequestException as e:
-    #         logger.error(f"Request failed on attempt {attempt+1}: {e}")
-    #         time.sleep(judge_config["wait_time"])
-    #         if attempt == judge_config["max_retries"] - 1:
-    #             logger.info(f"Failed to get response after {judge_config['max_retries']} attempts")
-    #             responses.append(None)
-    #     except Exception as e:
-    #         logger.info(f"Error on attempt {attempt+1}: {e}")
-    #         time.sleep(judge_config["wait_time"])
-    #         responses.append(None)
-
-    #Do the api call
-    response = api_call(api_url,headers,payload)
-    return response
+def pil_to_image_dict(pil_img):
+    """
+    Convert a PIL Image to an image dict with 'bytes' and 'path' keys.
+    """
+    buf = io.BytesIO()
+    pil_img.save(buf, format="PNG")  # or "JPEG" if you prefer
+    img_bytes = buf.getvalue()
+    return {'bytes': img_bytes, 'path': None}
 
-def api_call(api_url,headers,payload):
-    try:
-        response = requests.post(api_url, headers=headers, json=payload, timeout=60)
-        response.raise_for_status()  # Raises an HTTPError for bad responses (4xx, 5xx)
-        response_data = response.json()
-        return response_data
-        
-    except requests.exceptions.HTTPError as e:
-        logger.error(f"HTTP error occurred: {e}")
-        logger.error(f"Response status code: {e.response.status_code}")
-        logger.error(f"Response text: {e.response.text}")
-        return None
-        
-    except requests.exceptions.ConnectionError as e:
-        logger.error(f"Error connecting to the server: {e}")
-        return None
-        
-    except requests.exceptions.Timeout as e:
-        logger.error(f"Request timed out: {e}")
-        return None
-        
-    except requests.exceptions.RequestException as e:
-        logger.error(f"An error occurred while making the request: {e}")
-        return None
-        
-    except json.JSONDecodeError as e:
-        logger.error(f"Error decoding JSON response: {e}")
-        logger.error(f"Response text: {response.text}")
-        return None
-        
-    except Exception as e:
-        logger.error(f"An unexpected error occurred: {e}")
-        return None
 
-def process_judge_results(responses):
-    import pdb; pdb.set_trace()
-    for response in responses:
-        if response is None:
-            continue
-        response_data = response["choices"][0]["message"]["content"]
-        responses.append(response_data)
-    return responses
 
 def load_baseline_outputs(baseline_model_outputs_path):
     filtered_responses = []
@@ -182,24 +43,14 @@ def load_baseline_outputs(baseline_model_outputs_path):
     return filtered_responses
 
 
-def base64_to_bytes(base64_string):
-    # Remove the header if it exists (e.g., "data:image/jpeg;base64,")
-    if "base64," in base64_string:
-        base64_string = base64_string.split("base64,")[1]
-    # Decode base64 string to bytes
-    img_bytes = base64.b64decode(base64_string)
-    return img_bytes
-
 def process_docs(docs):
     """
     Process documents...
     """
     # logger.info(f"processing docs")
-    # Process images in place
-    docs = docs.select(range(5)) # filter out some samples!
-    # Create proper copies of images
+    # docs = docs.select(range(5)) # filter out some samples!
     def copy_image_fn(example):
-        example['copy_image'] = example['image'].copy()
+        example['copy_image'] = example['image']
         return example
     
     docs = docs.map(copy_image_fn)
@@ -225,40 +76,28 @@ def gen_doc_to_text(doc,lmms_eval_specific_kwargs=None ):
     return f"{pre_prompt}Question: {question}\n{post_prompt}"
 
 
-# def doc_to_choice(doc,lmms_eval_specific_kwargs=None ):
-#     choices = doc["choices"]
-#     choice_list = [f"{label}: {text}" for label, text in zip(choices["label"], choices["text"])]
-#     choice_list = choices["text"]
-#     return choice_list
-
-# def doc_to_target(doc,lmms_eval_specific_kwargs=None ):
-#     choices = doc["choices"]
-#     answerKey = doc["answerKey"]
-#     import pdb; pdb.set_trace()
-#     return choices["label"].index(answerKey)
-
 def gen_process_results(doc, results):
     generated_texts = results[0]
-    if isinstance(doc['copy_image'][0]['bytes'], bytes):
-        img = Image.open(io.BytesIO(doc['copy_image'][0]['bytes'])).convert('RGB')
-        myimg = img
+    myimg = doc['copy_image'][0]['bytes']
+    pil_img = Image.open(io.BytesIO(myimg))
     return {"results": {
         "id": doc["index"],
-        "image": myimg,
+        "image": pil_img,
         "question": doc["question"],
         "image_category": doc["image_source_category"],
         "prediction": generated_texts
-        } 
+        }
     }
 
 def aggregate_results(results):
     preds = [result["prediction"] for result in results]
     questions = [result["question"] for result in results]
     images = [result["image"] for result in results]
+    # Assuming images is a list of PIL Images:
+    images_bytes = [pil_to_image_dict(img) for img in images.copy()]
 
     judge_config = get_judge_config()
 
-    logger.info("Checking judge config...")
     if judge_config["run_judge"]:
         logger.info("Judge is enabled, checking judge config...")
         if judge_config["judge_prompt_type"] == "comparative":
@@ -267,17 +106,14 @@ def aggregate_results(results):
                 baseline_model_outputs = load_baseline_outputs(judge_config["baseline_model_outputs_path"])
                 assert len(baseline_model_outputs) == len(preds)
                 logger.info("Baseline model outputs loaded successfully.")
-                #TODO: run judge with comparative prompt
-                import pdb; pdb.set_trace()
-                judge_results = run_judge(questions,preds,judge_config,baseline_model_outputs,images)
+                judge_results = run_judge(questions,preds,judge_config,baseline_model_outputs,images_bytes)
             except Exception as e:
                 logger.error(f"Failed while loading model outputs or running judge.")
                 raise e
             
         elif judge_config["judge_prompt_type"] == "direct assessment":
             preds = [result["prediction"] for result in results]
-            #Note: run judge on predicted outputs
-            judge_results = run_judge(questions,preds,judge_config,baseline_model_outputs=None,images=images)
+            judge_results = run_judge(questions,preds,judge_config,baseline_model_outputs=None,images=images_bytes)
         else:
             logger.error(f"Invalid judge prompt type: {judge_config['judge_prompt_type']}.Skipping judge...")
             judge_results = None
@@ -289,15 +125,6 @@ def aggregate_results(results):
     if judge_results is None:
         return {"judge_results": None}
     else:
-        results = process_judge_results(judge_results)
+        results = compute_results(judge_results,judge_config)
         return results
 
-
-# def cache_judge_outputs(doc, round_res, previous_round_info, save_dir):
-#     save_dict = dict(
-#         sample_id=doc["index"],
-#         question=doc["question"],
-#         round_res=round_res,
-#     )
-#     save_dict.update(previous_round_info)
-#     json.dump(save_dict, open(os.path.join(save_dir, f"{save_dict['sample_id']}.json"), "w"), indent=4)

From a37a973ac6edd47edf3e557cede20306f92c8eb9 Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Wed, 16 Apr 2025 11:44:39 +0000
Subject: [PATCH 083/112] updated yamls for aya-vision-bench

---
 .../tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml    | 7 ++++++-
 .../tasks/ayavisionbench/aya-vision-bench-gen-de.yaml    | 7 ++++++-
 .../tasks/ayavisionbench/aya-vision-bench-gen-eng.yaml   | 1 +
 .../tasks/ayavisionbench/aya-vision-bench-gen-es.yaml    | 9 +++++++--
 .../tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml    | 8 +++++++-
 .../tasks/ayavisionbench/aya-vision-bench-gen-it.yaml    | 8 +++++++-
 .../tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml    | 8 +++++++-
 .../tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml    | 8 +++++++-
 .../tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml    | 8 +++++++-
 .../tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml    | 8 +++++++-
 .../tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml    | 8 +++++++-
 .../tasks/ayavisionbench/eval_with_judge_template.yaml   | 2 +-
 12 files changed, 70 insertions(+), 12 deletions(-)

diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml
index 25368377b..8f9fc7423 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml
@@ -3,9 +3,12 @@ dataset_name: ces_Latn
 task: aya-vision-bench-gen-cs
 test_split: test
 output_type: generate_until
+process_docs: !function utils.process_docs
 doc_to_visual: !function utils.gen_doc_to_visual
 doc_to_text: !function utils.gen_doc_to_text
 doc_to_target: ""
+process_results: !function utils.gen_process_results
+
 
 generation_kwargs:
   max_new_tokens: 1024
@@ -14,7 +17,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-process_results: !function utils.gen_process_results
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
 
 
 metadata:
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml
index 59cc4208c..00b6a76cc 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml
@@ -3,9 +3,12 @@ dataset_name: deu_Latn
 task: aya-vision-bench-gen-de
 test_split: test
 output_type: generate_until
+process_docs: !function utils.process_docs
 doc_to_visual: !function utils.gen_doc_to_visual
 doc_to_text: !function utils.gen_doc_to_text
 doc_to_target: ""
+process_results: !function utils.gen_process_results
+
 
 generation_kwargs:
   max_new_tokens: 1024
@@ -14,7 +17,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-process_results: !function utils.gen_process_results
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
 
 
 metadata:
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-eng.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-eng.yaml
index d0f3463bf..3bd92dcb4 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-eng.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-eng.yaml
@@ -10,6 +10,7 @@ doc_to_text: !function utils.gen_doc_to_text
 doc_to_target: ""
 process_results: !function utils.gen_process_results
 
+
 generation_kwargs:
   max_new_tokens: 1024
   temperature: 0
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml
index d31c4f996..611e576d5 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml
@@ -3,9 +3,13 @@ dataset_name: spa_Latn
 task: aya-vision-bench-gen-es
 test_split: test
 output_type: generate_until
+
+process_docs: !function utils.process_docs
 doc_to_visual: !function utils.gen_doc_to_visual
 doc_to_text: !function utils.gen_doc_to_text
 doc_to_target: ""
+process_results: !function utils.gen_process_results
+
 
 generation_kwargs:
   max_new_tokens: 1024
@@ -14,8 +18,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-process_results: !function utils.gen_process_results
-
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
 
 metadata:
   - version: 0.0
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml
index 34ffa0b10..78d979249 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml
@@ -3,9 +3,13 @@ dataset_name: fra_Latn
 task: aya-vision-bench-gen-fr
 test_split: test
 output_type: generate_until
+
+process_docs: !function utils.process_docs
 doc_to_visual: !function utils.gen_doc_to_visual
 doc_to_text: !function utils.gen_doc_to_text
 doc_to_target: ""
+process_results: !function utils.gen_process_results
+
 
 generation_kwargs:
   max_new_tokens: 1024
@@ -14,7 +18,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-process_results: !function utils.gen_process_results
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
 
 
 metadata:
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml
index 12c86333f..075ddcdd1 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml
@@ -3,9 +3,13 @@ dataset_name: ita_Latn
 task: aya-vision-bench-gen-it
 test_split: test
 output_type: generate_until
+
+process_docs: !function utils.process_docs
 doc_to_visual: !function utils.gen_doc_to_visual
 doc_to_text: !function utils.gen_doc_to_text
 doc_to_target: ""
+process_results: !function utils.gen_process_results
+
 
 generation_kwargs:
   max_new_tokens: 1024
@@ -14,7 +18,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-process_results: !function utils.gen_process_results
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
 
 
 metadata:
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml
index 620286bc2..755f5d4cc 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml
@@ -3,9 +3,13 @@ dataset_name: kor_Hang
 task: aya-vision-bench-gen-ko
 test_split: test
 output_type: generate_until
+
+process_docs: !function utils.process_docs
 doc_to_visual: !function utils.gen_doc_to_visual
 doc_to_text: !function utils.gen_doc_to_text
 doc_to_target: ""
+process_results: !function utils.gen_process_results
+
 
 generation_kwargs:
   max_new_tokens: 1024
@@ -14,7 +18,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-process_results: !function utils.gen_process_results
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
 
 
 metadata:
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml
index 6267d8c18..06edd841e 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml
@@ -3,9 +3,13 @@ dataset_name: nld_Latn
 task: aya-vision-bench-gen-nl
 test_split: test
 output_type: generate_until
+
+process_docs: !function utils.process_docs
 doc_to_visual: !function utils.gen_doc_to_visual
 doc_to_text: !function utils.gen_doc_to_text
 doc_to_target: ""
+process_results: !function utils.gen_process_results
+
 
 generation_kwargs:
   max_new_tokens: 1024
@@ -14,7 +18,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-process_results: !function utils.gen_process_results
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
 
 
 metadata:
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml
index afde9cb63..6f8a15f82 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml
@@ -3,9 +3,13 @@ dataset_name: por_Latn
 task: aya-vision-bench-gen-pt
 test_split: test
 output_type: generate_until
+
+process_docs: !function utils.process_docs
 doc_to_visual: !function utils.gen_doc_to_visual
 doc_to_text: !function utils.gen_doc_to_text
 doc_to_target: ""
+process_results: !function utils.gen_process_results
+
 
 generation_kwargs:
   max_new_tokens: 1024
@@ -14,7 +18,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-process_results: !function utils.gen_process_results
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
 
 
 metadata:
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml
index d2c30f97e..937127866 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml
@@ -3,9 +3,13 @@ dataset_name: rus_Cyrl
 task: aya-vision-bench-gen-ru
 test_split: test
 output_type: generate_until
+
+process_docs: !function utils.process_docs
 doc_to_visual: !function utils.gen_doc_to_visual
 doc_to_text: !function utils.gen_doc_to_text
 doc_to_target: ""
+process_results: !function utils.gen_process_results
+
 
 generation_kwargs:
   max_new_tokens: 1024
@@ -14,7 +18,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-process_results: !function utils.gen_process_results
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
 
 
 metadata:
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml
index 8ebb6fb34..b072f29ab 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml
@@ -3,9 +3,13 @@ dataset_name: zho_Hans
 task: aya-vision-bench-gen-zh
 test_split: test
 output_type: generate_until
+
+process_docs: !function utils.process_docs
 doc_to_visual: !function utils.gen_doc_to_visual
 doc_to_text: !function utils.gen_doc_to_text
 doc_to_target: ""
+process_results: !function utils.gen_process_results
+
 
 generation_kwargs:
   max_new_tokens: 1024
@@ -14,7 +18,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-process_results: !function utils.gen_process_results
+metric_list:
+  - metric: results
+    aggregation: !function utils.aggregate_results
 
 
 metadata:
diff --git a/lmms_eval/tasks/ayavisionbench/eval_with_judge_template.yaml b/lmms_eval/tasks/ayavisionbench/eval_with_judge_template.yaml
index 439aaa35c..d2757e2ec 100644
--- a/lmms_eval/tasks/ayavisionbench/eval_with_judge_template.yaml
+++ b/lmms_eval/tasks/ayavisionbench/eval_with_judge_template.yaml
@@ -1,7 +1,7 @@
 # if comparative, then the judge will compare the baseline model output with the predicted outputs. If the baseline model outputs path is not found, then the predicted outputs will be just saved, without running the judge.
 # if direct assessment, then the judge will just judge the predicted outputs!
 # to disable the judge, set run_judge to false
-run_judge: true
+run_judge: false
 api_type: openai #openai #anthropic
 judge_model_name: gpt-4o-mini #gpt-4o-mini #claude-3-7-sonnet-20250219
 judge_prompt_type: comparative

From d69bffa22cdf849135a08028a028a73b4e1b3ef7 Mon Sep 17 00:00:00 2001
From: aminfarajian <143111778+aminfarajian@users.noreply.github.com>
Date: Thu, 17 Apr 2025 14:41:30 +0100
Subject: [PATCH 084/112] Unify the argument name for the system prompt

---
 lmms_eval/models/llava.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lmms_eval/models/llava.py b/lmms_eval/models/llava.py
index 2ae9babad..dd0bceec4 100755
--- a/lmms_eval/models/llava.py
+++ b/lmms_eval/models/llava.py
@@ -65,7 +65,7 @@ def __init__(
         tie_weights: bool = True,
         truncate_context=False,  # whether to truncate the context in generation, set it False for LLaVA-1.6
         customized_config=None,  # ends in json
-        system_prompt="<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.",
+        add_system_prompt="<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.",
         **kwargs,
     ) -> None:
         super().__init__()
@@ -143,7 +143,7 @@ def __init__(
             self.model.to(self._device)
             self._rank = 0
             self._world_size = 1
-        self.system_prompt = system_prompt
+        self.system_prompt = add_system_prompt
     @property
     def config(self):
         # return the associated transformers.AutoConfig for the given pretrained model.

From 2d45f67c0661cad18b35bcbf1c0ecab5eb980701 Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Thu, 17 Apr 2025 19:02:48 +0000
Subject: [PATCH 085/112] changed to litellm usage for llms as judges for
 ayavisionbench

---
 .../ayavisionbench/eval_judge_template.yaml   |  16 ++
 .../eval_with_judge_template.yaml             |  12 --
 lmms_eval/tasks/ayavisionbench/judge_utils.py | 140 +++++++++++++++---
 lmms_eval/tasks/ayavisionbench/utils.py       |   2 +-
 4 files changed, 137 insertions(+), 33 deletions(-)
 create mode 100644 lmms_eval/tasks/ayavisionbench/eval_judge_template.yaml
 delete mode 100644 lmms_eval/tasks/ayavisionbench/eval_with_judge_template.yaml

diff --git a/lmms_eval/tasks/ayavisionbench/eval_judge_template.yaml b/lmms_eval/tasks/ayavisionbench/eval_judge_template.yaml
new file mode 100644
index 000000000..a2c3b6e57
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/eval_judge_template.yaml
@@ -0,0 +1,16 @@
+# if comparative, then the judge will compare the baseline model output with the predicted outputs. If the baseline model outputs path is not found, then the predicted outputs will be just saved, without running the judge.
+# if direct assessment, then the judge will just judge the predicted outputs!
+# to disable the judge, set run_judge to false
+run_judge: true
+api_type: openai #openai #anthropic
+judge_model_name: gpt-4o-mini  #gpt-4o #neulab/claude-3-7-sonnet-20250219  #claude-3-7-sonnet-20250219
+judge_prompt_type: comparative
+text_only: false
+baseline_model_outputs_path: /mnt/scratch-artemis/manos/data/tower-vision-eval-outputs/tower-vision/tower-vision-debug-results/aya-vision-bench-gen-eng/Unbabel__lnext-qwen2p5-7b-siglip2-v5/Unbabel__lnext-qwen2p5-7b-siglip2-v5/20250418_015501_samples_aya-vision-bench-gen-eng.jsonl #/mnt/scratch-artemis/manos/data/tower-vision-eval-outputs/tower-vision/tower-vision-debug-results/aya-vision-bench-gen-eng/Unbabel__lnext-qwen2p5-7b-siglip2-v5/Unbabel__lnext-qwen2p5-7b-siglip2-v5/20250416_060306_samples_aya-vision-bench-gen-eng.jsonl
+# max_tokens: 2048
+temperature: 0.0
+top_p: 1.0
+tensor_parallel_size: 1
+save_judge_parsed_outputs: true
+# api_url: https://api.openai.com/v1/chat/completions    #https://api.anthropic.com/v1/messages  for anthropic
+# api_url: https://cmu.litellm.ai # https://api.openai.com/v1/chat/completions
diff --git a/lmms_eval/tasks/ayavisionbench/eval_with_judge_template.yaml b/lmms_eval/tasks/ayavisionbench/eval_with_judge_template.yaml
deleted file mode 100644
index d2757e2ec..000000000
--- a/lmms_eval/tasks/ayavisionbench/eval_with_judge_template.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-# if comparative, then the judge will compare the baseline model output with the predicted outputs. If the baseline model outputs path is not found, then the predicted outputs will be just saved, without running the judge.
-# if direct assessment, then the judge will just judge the predicted outputs!
-# to disable the judge, set run_judge to false
-run_judge: false
-api_type: openai #openai #anthropic
-judge_model_name: gpt-4o-mini #gpt-4o-mini #claude-3-7-sonnet-20250219
-judge_prompt_type: comparative
-text_only: true
-baseline_model_outputs_path: /mnt/scratch-artemis/manos/data/tower-vision-eval-outputs/tower-vision/tower-vision-debug-results/aya-vision-bench-gen-eng/Unbabel__lnext-qwen2p5-7b-siglip2-v5/Unbabel__lnext-qwen2p5-7b-siglip2-v5/20250416_060306_samples_aya-vision-bench-gen-eng.jsonl
-max_tokens: 1024
-temperature: 0
-api_url: https://api.openai.com/v1/chat/completions    #https://api.anthropic.com/v1/messages  for anthropic
\ No newline at end of file
diff --git a/lmms_eval/tasks/ayavisionbench/judge_utils.py b/lmms_eval/tasks/ayavisionbench/judge_utils.py
index bcdbd8c5d..13950aae6 100644
--- a/lmms_eval/tasks/ayavisionbench/judge_utils.py
+++ b/lmms_eval/tasks/ayavisionbench/judge_utils.py
@@ -11,6 +11,8 @@
 from PIL import Image
 from tqdm import tqdm
 from typing import Dict, List, Optional, Union
+import litellm
+
 
 from lmms_eval.tasks.ayavisionbench.judge_templates import (
     COMPARATIVE_GEN_USER_PROMPT,
@@ -56,7 +58,7 @@ def img_bytes_to_url(image_dict: dict) -> str:
         return None
 
 def get_judge_config():
-    with open(Path(__file__).parent / "eval_with_judge_template.yaml", "r") as f:
+    with open(Path(__file__).parent / "eval_judge_template.yaml", "r") as f:
         raw_data = f.readlines()
         safe_data = []
         for i, line in enumerate(raw_data):
@@ -143,6 +145,66 @@ def set_prompts(judge_config,questions,preds,baseline_model_outputs=None):
     return system_prompt, prompts
 
 
+# def run_judge(
+#     questions: List[str],
+#     preds: List[str],
+#     judge_config: Dict,
+#     baseline_model_outputs: Optional[List[str]] = None,
+#     images: Optional[Dict[str, Union[bytes, None]]] = None
+# ) -> List[str]:
+#     """
+#     Run judge evaluation on predictions with optional image inputs.
+    
+#     Args:
+#         questions: List of questions to evaluate
+#         preds: List of predicted outputs to evaluate
+#         judge_config: Configuration dictionary for the judge
+#         baseline_model_outputs: Optional list of baseline outputs to compare against
+#         images: Optional dictionary containing image data with format {'bytes': bytes, 'path': None}
+        
+#     Returns:
+#         List[Dict]: List of parsed responses from the judge
+#     """
+#     logger.info(f"Selected judge type: {judge_config['judge_prompt_type']}")
+#     # # Load environment variables from .env file
+#     load_dotenv()
+    
+#     api_url = judge_config["api_url"]
+
+#     headers,payload = choose_api(api_url,judge_config)
+#     system_prompt, user_prompts = set_prompts(judge_config,questions,preds,baseline_model_outputs)
+    
+#     responses = []
+#     total_items = len(user_prompts)
+#     logger.info(f"Running judge evaluation for {total_items} items...")
+#     # Create progress bar with additional information
+#     with tqdm(total=total_items, 
+#               desc="Running judge evaluation", 
+#               bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]',
+#               ncols=100) as pbar:
+              
+#         for idx, (image, prompt) in enumerate(zip(images, user_prompts)):
+#             if image is not None or judge_config["text_only"]==False:
+#                 message=([
+#                         {"role": "system", "content": [{"type": "text", "text": system_prompt}]},
+#                         {"role": "user", "content": [{"type": "text", "text": prompt},{"type": "image_url", "image_url": {"url":img_bytes_to_url(image)}}]}
+#                         ])
+#             else:
+#                 message=[{"role": "system", "content": system_prompt},{"role": "user", "content": prompt}]
+            
+#             payload["messages"] = message
+
+#             response = api_call(api_url, headers, payload)
+#             responses.append(response)
+            
+#             # Update progress bar with current item
+#             pbar.set_postfix({"Current": f"{idx+1}/{total_items}"})
+#             pbar.update(1)
+    
+#     responses = parse_judge_responses(responses,judge_config)
+#     return responses
+
+
 def run_judge(
     questions: List[str],
     preds: List[str],
@@ -151,7 +213,7 @@ def run_judge(
     images: Optional[Dict[str, Union[bytes, None]]] = None
 ) -> List[str]:
     """
-    Run judge evaluation on predictions with optional image inputs.
+    Run judge evaluation on predictions using LiteLLM.
     
     Args:
         questions: List of questions to evaluate
@@ -164,45 +226,81 @@ def run_judge(
         List[Dict]: List of parsed responses from the judge
     """
     logger.info(f"Selected judge type: {judge_config['judge_prompt_type']}")
-    # # Load environment variables from .env file
     load_dotenv()
     
-    api_url = judge_config["api_url"]
+    # Set up model configuration
+    model = judge_config["judge_model_name"]
+    
+
+    
+    if os.getenv("LITELLM_API_KEY"):
+        api_key = os.getenv("LITELLM_API_KEY")
+        # api_url = "https://cmu.litellm.ai"
+    elif os.getenv("ANTHROPIC_API_KEY"):
+        api_key = os.getenv("ANTHROPIC_API_KEY")
+        # api_url = "https://api.anthropic.com/v1/messages"
+    elif os.getenv("OPENAI_API_KEY"):
+        api_key = os.getenv("OPENAI_API_KEY")
+        # api_url = "https://api.openai.com/v1/chat/completions"
+    else:
+        raise ValueError("No API key found. Please set LITELLM_API_KEY, ANTHROPIC_API_KEY, or OPENAI_API_KEY environment variable or define the api_key in the judge_config")
+
+    
+
+    # if judge_config["text_only"] == False:
+    #     assert litellm.supports_vision(model=model),f"Selected judge model:{model} does not support vision"
 
-    headers,payload = choose_api(api_url,judge_config)
-    system_prompt, user_prompts = set_prompts(judge_config,questions,preds,baseline_model_outputs)
+    # Get prompts
+    system_prompt, user_prompts = set_prompts(judge_config, questions, preds, baseline_model_outputs)
     
     responses = []
     total_items = len(user_prompts)
     logger.info(f"Running judge evaluation for {total_items} items...")
-    # Create progress bar with additional information
+    
     with tqdm(total=total_items, 
               desc="Running judge evaluation", 
               bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]',
               ncols=100) as pbar:
               
         for idx, (image, prompt) in enumerate(zip(images, user_prompts)):
-            if image is not None or judge_config["text_only"]==False:
-                message=([
+            try:
+                # Prepare messages based on whether image is present
+                if image is not None or judge_config["text_only"] == False:
+                    messages = [
                         {"role": "system", "content": [{"type": "text", "text": system_prompt}]},
-                        {"role": "user", "content": [{"type": "text", "text": prompt},{"type": "image_url", "image_url": {"url":img_bytes_to_url(image)}}]}
-                        ])
-            else:
-                message=[{"role": "system", "content": system_prompt},{"role": "user", "content": prompt}]
-            
-            payload["messages"] = message
+                        {"role": "user", "content": [
+                            {"type": "text", "text": prompt},
+                            {"type": "image_url", "image_url": {"url": img_bytes_to_url(image)}}
+                        ]}
+                    ]
+                else:
+                    messages = [
+                        {"role": "system", "content": system_prompt},
+                        {"role": "user", "content": prompt}
+                    ]
 
-            response = api_call(api_url, headers, payload)
-            responses.append(response)
+                # Make the LiteLLM call
+                response = litellm.completion(
+                    model=model,
+                    messages=messages,
+                    max_tokens= judge_config["max_tokens"] if "max_tokens" in judge_config else None,
+                    temperature=judge_config["temperature"] if "temperature" in judge_config else None,
+                    top_p=judge_config["top_p"] if "top_p" in judge_config else None,
+                    api_key=api_key,
+                    base_url=judge_config["api_url"] if "api_url" in judge_config else None
+                )
+                responses.append(response)
+                
+            except Exception as e:
+                logger.error(f"Error in LiteLLM call: {str(e)}")
+                responses.append(None)
             
-            # Update progress bar with current item
             pbar.set_postfix({"Current": f"{idx+1}/{total_items}"})
             pbar.update(1)
     
-    responses = parse_judge_responses(responses,judge_config)
+    responses = parse_judge_responses(responses, judge_config)
     return responses
 
-
 def parse_comparative_response(response: str) -> str:
     """
     Extract verdict pattern from judge response.
@@ -269,4 +367,6 @@ def compute_results(responses,judge_config):
         }
     elif judge_config["judge_prompt_type"] == "direct_assessment":
         raise NotImplementedError("Direct assessment results computation not implemented yet!")
+    if judge_config["save_judge_parsed_outputs"]:
+        results["judge_parsed_outputs"] = responses
     return results
\ No newline at end of file
diff --git a/lmms_eval/tasks/ayavisionbench/utils.py b/lmms_eval/tasks/ayavisionbench/utils.py
index bff95c4fa..23f211d47 100644
--- a/lmms_eval/tasks/ayavisionbench/utils.py
+++ b/lmms_eval/tasks/ayavisionbench/utils.py
@@ -48,7 +48,7 @@ def process_docs(docs):
     Process documents...
     """
     # logger.info(f"processing docs")
-    # docs = docs.select(range(5)) # filter out some samples!
+    docs = docs.select(range(5)) # filter out some samples!
     def copy_image_fn(example):
         example['copy_image'] = example['image']
         return example

From 15377a9dba826677fd976974981db8fa40ef35f5 Mon Sep 17 00:00:00 2001
From: aminfarajian <143111778+aminfarajian@users.noreply.github.com>
Date: Mon, 21 Apr 2025 17:30:24 +0100
Subject: [PATCH 086/112] Update llava.py

Rollback to the default system prompt for llava.
---
 lmms_eval/models/llava.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lmms_eval/models/llava.py b/lmms_eval/models/llava.py
index dd0bceec4..f47839e92 100755
--- a/lmms_eval/models/llava.py
+++ b/lmms_eval/models/llava.py
@@ -65,7 +65,7 @@ def __init__(
         tie_weights: bool = True,
         truncate_context=False,  # whether to truncate the context in generation, set it False for LLaVA-1.6
         customized_config=None,  # ends in json
-        add_system_prompt="<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.",
+        add_system_prompt=None,
         **kwargs,
     ) -> None:
         super().__init__()
@@ -143,7 +143,7 @@ def __init__(
             self.model.to(self._device)
             self._rank = 0
             self._world_size = 1
-        self.system_prompt = add_system_prompt
+        self.add_system_prompt = add_system_prompt
     @property
     def config(self):
         # return the associated transformers.AutoConfig for the given pretrained model.
@@ -373,7 +373,7 @@ def _collate(x):
                     conv = conv_templates[self.conv_template].copy()
                 # if conv.system:
                 #     conv.system = self.add_system_prompt
-                if self.system_prompt and conv.system:
+                if self.add_system_prompt and conv.system:
                     conv.system = self.system_prompt
                 conv.append_message(conv.roles[0], question)
                 conv.append_message(conv.roles[1], None)

From 16a1908134559fb9e268de088f2782036f06cb37 Mon Sep 17 00:00:00 2001
From: farajian <amin@unbabel.com>
Date: Mon, 21 Apr 2025 17:27:23 +0000
Subject: [PATCH 087/112] small fixes in judge_utils.py

---
 lmms_eval/tasks/ayavisionbench/judge_utils.py | 27 +++++++++++--------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/lmms_eval/tasks/ayavisionbench/judge_utils.py b/lmms_eval/tasks/ayavisionbench/judge_utils.py
index 13950aae6..45431d1de 100644
--- a/lmms_eval/tasks/ayavisionbench/judge_utils.py
+++ b/lmms_eval/tasks/ayavisionbench/judge_utils.py
@@ -232,16 +232,21 @@ def run_judge(
     model = judge_config["judge_model_name"]
     
 
-    
-    if os.getenv("LITELLM_API_KEY"):
-        api_key = os.getenv("LITELLM_API_KEY")
-        # api_url = "https://cmu.litellm.ai"
-    elif os.getenv("ANTHROPIC_API_KEY"):
-        api_key = os.getenv("ANTHROPIC_API_KEY")
-        # api_url = "https://api.anthropic.com/v1/messages"
-    elif os.getenv("OPENAI_API_KEY"):
-        api_key = os.getenv("OPENAI_API_KEY")
-        # api_url = "https://api.openai.com/v1/chat/completions"
+    if judge_config["api_type"] == "anthropic":
+        if os.getenv("ANTHROPIC_API_KEY"):
+            api_key = os.getenv("ANTHROPIC_API_KEY")
+        else:
+            raise ValueError("No API key found. Please set ANTHROPIC_API_KEY environment variable or define the api_key in the judge_config")
+    elif judge_config["api_type"] == "openai":
+        if os.getenv("OPENAI_API_KEY"):
+            api_key = os.getenv("OPENAI_API_KEY")
+        else:
+            raise ValueError("No API key found. Please set OPENAI_API_KEY environment variable or define the api_key in the judge_config")
+    elif judge_config["api_type"] == "litellm":
+        if os.getenv("LITELLM_API_KEY"):
+            api_key = os.getenv("LITELLM_API_KEY")
+        else:
+            raise ValueError("No API key found. Please set LITELLM_API_KEY environment variable or define the api_key in the judge_config")
     else:
         raise ValueError("No API key found. Please set LITELLM_API_KEY, ANTHROPIC_API_KEY, or OPENAI_API_KEY environment variable or define the api_key in the judge_config")
 
@@ -287,7 +292,7 @@ def run_judge(
                     temperature=judge_config["temperature"] if "temperature" in judge_config else None,
                     top_p=judge_config["top_p"] if "top_p" in judge_config else None,
                     api_key=api_key,
-                    base_url=judge_config["api_url"] if "api_url" in judge_config else None
+                    base_url=judge_config.get("api_url")
                 )
                 responses.append(response)
                 

From 19da9a61c3be0def59e3147a1561625392c5b84d Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Sun, 27 Apr 2025 14:17:25 +0000
Subject: [PATCH 088/112] updated aya vision with independent judge script

---
 .../aya-vision-bench-gen-cs.yaml              |   6 +-
 .../aya-vision-bench-gen-de.yaml              |   6 +-
 .../aya-vision-bench-gen-eng.yaml             |   6 +-
 .../aya-vision-bench-gen-es.yaml              |   6 +-
 .../aya-vision-bench-gen-fr.yaml              |   6 +-
 .../aya-vision-bench-gen-it.yaml              |   6 +-
 .../aya-vision-bench-gen-ko.yaml              |   6 +-
 .../aya-vision-bench-gen-nl.yaml              |   6 +-
 .../aya-vision-bench-gen-pt.yaml              |   6 +-
 .../aya-vision-bench-gen-ru.yaml              |   6 +-
 .../aya-vision-bench-gen-zh.yaml              |   6 +-
 .../ayavisionbench/eval_judge_template.yaml   |  22 +--
 lmms_eval/tasks/ayavisionbench/judge_eval.py  | 151 ++++++++++++++++
 lmms_eval/tasks/ayavisionbench/judge_utils.py | 162 +++---------------
 lmms_eval/tasks/ayavisionbench/utils.py       |  75 ++++----
 15 files changed, 254 insertions(+), 222 deletions(-)
 create mode 100644 lmms_eval/tasks/ayavisionbench/judge_eval.py

diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml
index 8f9fc7423..d35aca642 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml
@@ -17,9 +17,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
 
 
 metadata:
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml
index 00b6a76cc..043af92fb 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml
@@ -17,9 +17,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
 
 
 metadata:
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-eng.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-eng.yaml
index 3bd92dcb4..40a427bc8 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-eng.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-eng.yaml
@@ -18,9 +18,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
 
 metadata:
   - version: 0.0
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml
index 611e576d5..df13935b1 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml
@@ -18,9 +18,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
 
 metadata:
   - version: 0.0
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml
index 78d979249..3e624eef5 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml
@@ -18,9 +18,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
 
 
 metadata:
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml
index 075ddcdd1..110e4fc7b 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml
@@ -18,9 +18,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
 
 
 metadata:
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml
index 755f5d4cc..12058e158 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml
@@ -18,9 +18,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
 
 
 metadata:
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml
index 06edd841e..c8b1f2c88 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml
@@ -18,9 +18,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
 
 
 metadata:
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml
index 6f8a15f82..96b8c8952 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml
@@ -18,9 +18,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
 
 
 metadata:
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml
index 937127866..8eabc0b1b 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml
@@ -18,9 +18,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
 
 
 metadata:
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml
index b072f29ab..9c5093878 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml
@@ -18,9 +18,9 @@ generation_kwargs:
   num_beams: 1
   do_sample: false
 
-metric_list:
-  - metric: results
-    aggregation: !function utils.aggregate_results
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
 
 
 metadata:
diff --git a/lmms_eval/tasks/ayavisionbench/eval_judge_template.yaml b/lmms_eval/tasks/ayavisionbench/eval_judge_template.yaml
index a2c3b6e57..fcf0e0a8a 100644
--- a/lmms_eval/tasks/ayavisionbench/eval_judge_template.yaml
+++ b/lmms_eval/tasks/ayavisionbench/eval_judge_template.yaml
@@ -1,16 +1,16 @@
 # if comparative, then the judge will compare the baseline model output with the predicted outputs. If the baseline model outputs path is not found, then the predicted outputs will be just saved, without running the judge.
 # if direct assessment, then the judge will just judge the predicted outputs!
 # to disable the judge, set run_judge to false
-run_judge: true
-api_type: openai #openai #anthropic
-judge_model_name: gpt-4o-mini  #gpt-4o #neulab/claude-3-7-sonnet-20250219  #claude-3-7-sonnet-20250219
-judge_prompt_type: comparative
-text_only: false
-baseline_model_outputs_path: /mnt/scratch-artemis/manos/data/tower-vision-eval-outputs/tower-vision/tower-vision-debug-results/aya-vision-bench-gen-eng/Unbabel__lnext-qwen2p5-7b-siglip2-v5/Unbabel__lnext-qwen2p5-7b-siglip2-v5/20250418_015501_samples_aya-vision-bench-gen-eng.jsonl #/mnt/scratch-artemis/manos/data/tower-vision-eval-outputs/tower-vision/tower-vision-debug-results/aya-vision-bench-gen-eng/Unbabel__lnext-qwen2p5-7b-siglip2-v5/Unbabel__lnext-qwen2p5-7b-siglip2-v5/20250416_060306_samples_aya-vision-bench-gen-eng.jsonl
-# max_tokens: 2048
-temperature: 0.0
-top_p: 1.0
-tensor_parallel_size: 1
-save_judge_parsed_outputs: true
+# run_judge: false
+# api_type: openai #openai #anthropic
+# judge_model_name: gpt-4o-mini  #gpt-4o #neulab/claude-3-7-sonnet-20250219  #claude-3-7-sonnet-20250219
+# judge_prompt_type: comparative
+# text_only: false
+# baseline_model_outputs_path: /mnt/scratch-artemis/manos/data/tower-vision-eval-outputs/tower-vision/tower-vision-debug-results/aya-vision-bench-gen-eng/Unbabel__lnext-qwen2p5-7b-siglip2-v5/Unbabel__lnext-qwen2p5-7b-siglip2-v5/20250418_015501_samples_aya-vision-bench-gen-eng.jsonl #/mnt/scratch-artemis/manos/data/tower-vision-eval-outputs/tower-vision/tower-vision-debug-results/aya-vision-bench-gen-eng/Unbabel__lnext-qwen2p5-7b-siglip2-v5/Unbabel__lnext-qwen2p5-7b-siglip2-v5/20250416_060306_samples_aya-vision-bench-gen-eng.jsonl
+# # max_tokens: 2048
+# temperature: 0.0
+# top_p: 1.0
+# tensor_parallel_size: 1
+# save_judge_parsed_outputs: true
 # api_url: https://api.openai.com/v1/chat/completions    #https://api.anthropic.com/v1/messages  for anthropic
 # api_url: https://cmu.litellm.ai # https://api.openai.com/v1/chat/completions
diff --git a/lmms_eval/tasks/ayavisionbench/judge_eval.py b/lmms_eval/tasks/ayavisionbench/judge_eval.py
new file mode 100644
index 000000000..4b9860a82
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/judge_eval.py
@@ -0,0 +1,151 @@
+import argparse
+import json
+from PIL import Image
+import os
+import io
+from loguru import logger
+
+from datasets import load_dataset
+from lmms_eval.tasks.ayavisionbench.judge_utils import run_judge,compute_results,parse_judge_responses
+
+def load_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--api_type",choices=["openai", "anthropic","litellm"], type=str, required=True)
+    parser.add_argument("--judge_model_name",default="gpt-4o-mini", type=str, required=True)
+    parser.add_argument("--judge_prompt_type",default="comparative",choices=["comparative", "direct assessment"], type=str, required=True)
+    parser.add_argument("--text_only",default=False, type=bool, required=True,help="if true, the judge model will not see the image.")
+    parser.add_argument("--model_output_path", type=str, required=True)
+    parser.add_argument("--baseline_output_path", type=str, required=True)
+    parser.add_argument("--lp", type=str, required=True)
+    parser.add_argument("--output_dir", type=str, required=True)
+    parser.add_argument("--save_judge_parsed_outputs",default=True, type=bool, required=True,help="if true, the judge parsed outputs will be saved.")
+    parser.add_argument("--save_judge_full_responses",default=True, type=bool, required=True,help="if true, the judge full responses will be saved.")
+    parser.add_argument("--save_judge_results",default=True, type=bool, required=True,help="if true, the judge results will be saved.")
+    parser.add_argument("--max_tokens",default=2048, type=int, required=True,help="the max tokens for the judge model.")
+    parser.add_argument("--temperature",default=0.0, type=float, required=True,help="the temperature for the judge model.")
+    parser.add_argument("--top_p",default=1.0, type=float, required=True,help="the top p for the judge model.")
+    parser.add_argument("--tensor_parallel_size",default=1, type=int, required=True,help="the tensor parallel size for the judge model.")
+    args = parser.parse_args()
+    return args
+
+def pil_to_image_dict(pil_img):
+    """
+    Convert a PIL Image to an image dict with 'bytes' and 'path' keys.
+    """
+    buf = io.BytesIO()
+    pil_img.save(buf, format="PNG")  # or "JPEG" if you prefer
+    img_bytes = buf.getvalue()
+    return {'bytes': img_bytes, 'path': None}
+
+def load_aya_vision_bench_data(language):
+    mapping_language_to_dataset_name = {
+        "cs": "ces_Latn",
+        "eng": "eng_Latn",
+        "fr": "fra_Latn",
+        "de": "deu_Latn",
+        "es": "spa_Latn",
+        "it": "ita_Latn",
+        "ko": "kor_Hang",
+        "nl": "nld_Latn",
+        "pt": "por_Latn",
+        "ru": "rus_Cyrl",
+        "zh": "zho_Hans",
+    }
+    dataset = load_dataset("CohereForAI/AyaVisionBench",name=mapping_language_to_dataset_name[language],split="test")
+    logger.info(f"Loaded Aya Vision Bench dataset for language: {language}")
+    dataset = dataset.select(range(5))
+    # Extract questions and images from the dataset
+    questions = [item["question"] for item in dataset]
+
+    images_bytes = []
+    questions = []
+    for item in dataset:
+        questions.append(item["question"])
+        pil_img = item["image"][0]
+        images_bytes.append(pil_to_image_dict(pil_img))
+    return dataset,questions,images_bytes
+
+def load_model_outputs(model_outputs_path):
+    filtered_responses = []
+    with open(model_outputs_path, 'r', encoding='utf-8') as f:
+        for line in f:
+            try:
+                data = json.loads(line.strip())
+                if 'filtered_resps' in data:
+                    # Handle both single string and list of strings cases
+                    resps = data['filtered_resps']
+                    if isinstance(resps, list):
+                        filtered_responses.extend(resps)
+                    else:
+                        filtered_responses.append(resps)
+            except json.JSONDecodeError:
+                print(f"Warning: Skipping invalid JSON line")
+                continue
+    return filtered_responses
+
+def extract_judge_message(response):
+    """
+    Extracts the output message content from a judge response object.
+    Assumes response.choices[0].message.content exists.
+    """
+    try:
+        return response.choices[0].message.content
+    except (AttributeError, IndexError, KeyError, TypeError):
+        return None
+
+if __name__ == "__main__":
+    
+    args = load_args()
+
+    # 1. Load Aya Vision Bench data
+    aya_dataset,questions,images_bytes = load_aya_vision_bench_data(args.lp)
+
+    # 2. Load model outputs
+    baseline_model_outputs = load_model_outputs(args.baseline_output_path)
+    model_outputs = load_model_outputs(args.model_output_path)
+    logger.info(f"Loaded baseline and model outputs for language: {args.lp}")
+    
+    # 3. Prepare judge_config dictionary
+    judge_config = {
+        "api_type": args.api_type,
+        "judge_model_name": args.judge_model_name,
+        "judge_prompt_type": args.judge_prompt_type,
+        "text_only": args.text_only,
+        # "api_url": os.environ.get("API_URL", None),  # or set as needed
+        "max_tokens": args.max_tokens,
+        "temperature": args.temperature,
+        "top_p": args.top_p,
+        "tensor_parallel_size": args.tensor_parallel_size,
+        # Add any other config keys your run_judge expects
+    }
+
+    # 4. Run judge
+    logger.info(f"Running judge for language: {args.lp}")
+    full_responses = run_judge(questions, model_outputs, judge_config, baseline_model_outputs, images_bytes)
+    f_resp = [extract_judge_message(resp) for resp in full_responses]
+
+    logger.info(f"Judge completed for language: {args.lp}")
+    parsed_responses = parse_judge_responses(full_responses, judge_config)
+    logger.info(f"Parsed responses completed for language: {args.lp}")
+    
+    # 7. Compute results
+    results = compute_results(parsed_responses, judge_config)
+    breakpoint()
+    # 5. Save results
+    if args.save_judge_parsed_outputs:
+        with open(os.path.join(args.output_dir, "judge_results_parsed.json"), "w") as f:
+            logger.info(f"Saving parsed responses...")
+            json.dump(parsed_responses, f)
+
+    if args.save_judge_full_responses:
+        with open(os.path.join(args.output_dir, "judge_results_full.json"), "w") as f:
+            logger.info(f"Saving full responses...")
+            json.dump(f_resp, f)
+
+    if args.save_judge_results:
+        with open(os.path.join(args.output_dir, "judge_results.json"), "w") as f:
+            logger.info(f"Saving results...")
+            json.dump(results, f)
+
+    
+    
\ No newline at end of file
diff --git a/lmms_eval/tasks/ayavisionbench/judge_utils.py b/lmms_eval/tasks/ayavisionbench/judge_utils.py
index 13950aae6..3052841c9 100644
--- a/lmms_eval/tasks/ayavisionbench/judge_utils.py
+++ b/lmms_eval/tasks/ayavisionbench/judge_utils.py
@@ -69,68 +69,6 @@ def get_judge_config():
     return config
 
 
-def choose_api(api_url,judge_config):
-    judge_model_name = judge_config["judge_model_name"]
-    if judge_config["api_type"] == "anthropic":
-        api_key = os.getenv('ANTHROPIC_API_KEY')
-        headers = {
-        "x-api-key": api_key,  # Different header for Claude
-        "Content-Type": "application/json",
-        "anthropic-version": "2023-06-01"  # Required for Claude
-    }
-    elif judge_config["api_type"] == "openai":
-        api_key = os.getenv('OPENAI_API_KEY')
-        headers = {
-            "Authorization": f"Bearer {api_key}",
-            "Content-Type": "application/json",
-        }
-    else:
-        logger.error(f"API type not supported. Please implement it!")
-        raise ValueError(f"Invalid API type: {judge_config['api_type']}")
-
-    payload = {
-        "model": judge_model_name,
-        "max_tokens": judge_config["max_tokens"],
-        "temperature": judge_config["temperature"],
-    }
-    return headers,payload
-
-
-def api_call(api_url,headers,payload):
-    try:
-        response = requests.post(api_url, headers=headers, json=payload, timeout=60)
-        response.raise_for_status()  # Raises an HTTPError for bad responses (4xx, 5xx)
-        response_data = response.json()
-        return response_data
-        
-    except requests.exceptions.HTTPError as e:
-        logger.error(f"HTTP error occurred: {e}")
-        logger.error(f"Response status code: {e.response.status_code}")
-        logger.error(f"Response text: {e.response.text}")
-        return None
-        
-    except requests.exceptions.ConnectionError as e:
-        logger.error(f"Error connecting to the server: {e}")
-        return None
-        
-    except requests.exceptions.Timeout as e:
-        logger.error(f"Request timed out: {e}")
-        return None
-        
-    except requests.exceptions.RequestException as e:
-        logger.error(f"An error occurred while making the request: {e}")
-        return None
-        
-    except json.JSONDecodeError as e:
-        logger.error(f"Error decoding JSON response: {e}")
-        logger.error(f"Response text: {response.text}")
-        return None
-        
-    except Exception as e:
-        logger.error(f"An unexpected error occurred: {e}")
-        return None
-
-
 def set_prompts(judge_config,questions,preds,baseline_model_outputs=None):
     if judge_config["judge_prompt_type"] == "comparative":
         system_prompt = COMPARATIVE_GEN_SYSTEM_PROMPT
@@ -145,72 +83,13 @@ def set_prompts(judge_config,questions,preds,baseline_model_outputs=None):
     return system_prompt, prompts
 
 
-# def run_judge(
-#     questions: List[str],
-#     preds: List[str],
-#     judge_config: Dict,
-#     baseline_model_outputs: Optional[List[str]] = None,
-#     images: Optional[Dict[str, Union[bytes, None]]] = None
-# ) -> List[str]:
-#     """
-#     Run judge evaluation on predictions with optional image inputs.
-    
-#     Args:
-#         questions: List of questions to evaluate
-#         preds: List of predicted outputs to evaluate
-#         judge_config: Configuration dictionary for the judge
-#         baseline_model_outputs: Optional list of baseline outputs to compare against
-#         images: Optional dictionary containing image data with format {'bytes': bytes, 'path': None}
-        
-#     Returns:
-#         List[Dict]: List of parsed responses from the judge
-#     """
-#     logger.info(f"Selected judge type: {judge_config['judge_prompt_type']}")
-#     # # Load environment variables from .env file
-#     load_dotenv()
-    
-#     api_url = judge_config["api_url"]
-
-#     headers,payload = choose_api(api_url,judge_config)
-#     system_prompt, user_prompts = set_prompts(judge_config,questions,preds,baseline_model_outputs)
-    
-#     responses = []
-#     total_items = len(user_prompts)
-#     logger.info(f"Running judge evaluation for {total_items} items...")
-#     # Create progress bar with additional information
-#     with tqdm(total=total_items, 
-#               desc="Running judge evaluation", 
-#               bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]',
-#               ncols=100) as pbar:
-              
-#         for idx, (image, prompt) in enumerate(zip(images, user_prompts)):
-#             if image is not None or judge_config["text_only"]==False:
-#                 message=([
-#                         {"role": "system", "content": [{"type": "text", "text": system_prompt}]},
-#                         {"role": "user", "content": [{"type": "text", "text": prompt},{"type": "image_url", "image_url": {"url":img_bytes_to_url(image)}}]}
-#                         ])
-#             else:
-#                 message=[{"role": "system", "content": system_prompt},{"role": "user", "content": prompt}]
-            
-#             payload["messages"] = message
-
-#             response = api_call(api_url, headers, payload)
-#             responses.append(response)
-            
-#             # Update progress bar with current item
-#             pbar.set_postfix({"Current": f"{idx+1}/{total_items}"})
-#             pbar.update(1)
-    
-#     responses = parse_judge_responses(responses,judge_config)
-#     return responses
-
 
 def run_judge(
     questions: List[str],
     preds: List[str],
     judge_config: Dict,
     baseline_model_outputs: Optional[List[str]] = None,
-    images: Optional[Dict[str, Union[bytes, None]]] = None
+    images: Optional[Dict[str, Union[bytes, None]]] = None,
 ) -> List[str]:
     """
     Run judge evaluation on predictions using LiteLLM.
@@ -232,16 +111,19 @@ def run_judge(
     model = judge_config["judge_model_name"]
     
 
-    
-    if os.getenv("LITELLM_API_KEY"):
-        api_key = os.getenv("LITELLM_API_KEY")
-        # api_url = "https://cmu.litellm.ai"
-    elif os.getenv("ANTHROPIC_API_KEY"):
-        api_key = os.getenv("ANTHROPIC_API_KEY")
-        # api_url = "https://api.anthropic.com/v1/messages"
-    elif os.getenv("OPENAI_API_KEY"):
-        api_key = os.getenv("OPENAI_API_KEY")
-        # api_url = "https://api.openai.com/v1/chat/completions"
+    if judge_config["api_type"] == "openai":
+        if os.getenv("OPENAI_API_KEY"):
+            api_key = os.getenv("OPENAI_API_KEY")
+        else:
+            api_key = judge_config["openai_api_key"]
+    elif judge_config["api_type"] == "anthropic":
+        if os.getenv("ANTHROPIC_API_KEY"):
+            api_key = os.getenv("ANTHROPIC_API_KEY")
+        else:
+            api_key = judge_config["anthropic_api_key"]
+    elif judge_config["api_type"] == "litellm":
+        if os.getenv("LITELLM_API_KEY"):
+            api_key = os.getenv("LITELLM_API_KEY")
     else:
         raise ValueError("No API key found. Please set LITELLM_API_KEY, ANTHROPIC_API_KEY, or OPENAI_API_KEY environment variable or define the api_key in the judge_config")
 
@@ -297,8 +179,6 @@ def run_judge(
             
             pbar.set_postfix({"Current": f"{idx+1}/{total_items}"})
             pbar.update(1)
-    
-    responses = parse_judge_responses(responses, judge_config)
     return responses
 
 def parse_comparative_response(response: str) -> str:
@@ -358,15 +238,15 @@ def compute_results(responses,judge_config):
             else:
                 no_answer += 1
         results = {
-            "a_better_than_b": a_better_than_b/len(responses),
-            "a_significantly_better_than_b": a_significantly_better_than_b/len(responses),
-            "b_better_than_a": b_better_than_a/len(responses),
-            "b_significantly_better_than_a": b_significantly_better_than_a/len(responses),
-            "a_equal_to_b": a_equal_to_b/len(responses),
+            "baseline_better_than_model": a_better_than_b/len(responses),
+            "baseline_significantly_better_than_model": a_significantly_better_than_b/len(responses),
+            "model_better_than_baseline": b_better_than_a/len(responses),
+            "model_significantly_better_than_baseline": b_significantly_better_than_a/len(responses),
+            "model_equal_to_baseline": a_equal_to_b/len(responses),
             "no_answer_matched": no_answer/len(responses)
         }
     elif judge_config["judge_prompt_type"] == "direct_assessment":
         raise NotImplementedError("Direct assessment results computation not implemented yet!")
-    if judge_config["save_judge_parsed_outputs"]:
-        results["judge_parsed_outputs"] = responses
+    else:
+        raise ValueError(f"Invalid judge prompt type: {judge_config['judge_prompt_type']}")
     return results
\ No newline at end of file
diff --git a/lmms_eval/tasks/ayavisionbench/utils.py b/lmms_eval/tasks/ayavisionbench/utils.py
index 23f211d47..060adbd0d 100644
--- a/lmms_eval/tasks/ayavisionbench/utils.py
+++ b/lmms_eval/tasks/ayavisionbench/utils.py
@@ -48,7 +48,7 @@ def process_docs(docs):
     Process documents...
     """
     # logger.info(f"processing docs")
-    docs = docs.select(range(5)) # filter out some samples!
+    # docs = docs.select(range(5)) # filter out some samples!
     def copy_image_fn(example):
         example['copy_image'] = example['image']
         return example
@@ -89,42 +89,43 @@ def gen_process_results(doc, results):
         }
     }
 
-def aggregate_results(results):
-    preds = [result["prediction"] for result in results]
-    questions = [result["question"] for result in results]
-    images = [result["image"] for result in results]
-    # Assuming images is a list of PIL Images:
-    images_bytes = [pil_to_image_dict(img) for img in images.copy()]
-
-    judge_config = get_judge_config()
-
-    if judge_config["run_judge"]:
-        logger.info("Judge is enabled, checking judge config...")
-        if judge_config["judge_prompt_type"] == "comparative":
-            logger.info("Judgement is set to comparative, checking baseline model outputs...")
-            try:
-                baseline_model_outputs = load_baseline_outputs(judge_config["baseline_model_outputs_path"])
-                assert len(baseline_model_outputs) == len(preds)
-                logger.info("Baseline model outputs loaded successfully.")
-                judge_results = run_judge(questions,preds,judge_config,baseline_model_outputs,images_bytes)
-            except Exception as e:
-                logger.error(f"Failed while loading model outputs or running judge.")
-                raise e
+# def aggregate_results(results):
+#     # preds = [result["prediction"] for result in results]
+#     # questions = [result["question"] for result in results]
+#     # images = [result["image"] for result in results]
+#     # # Assuming images is a list of PIL Images:
+#     # images_bytes = [pil_to_image_dict(img) for img in images.copy()]
+
+#     # judge_config = get_judge_config()
+
+#     # if judge_config["run_judge"]:
+#     #     logger.info("Judge is enabled, checking judge config...")
+#     #     if judge_config["judge_prompt_type"] == "comparative":
+#     #         logger.info("Judgement is set to comparative, checking baseline model outputs...")
+#     #         try:
+#     #             baseline_model_outputs = load_baseline_outputs(judge_config["baseline_model_outputs_path"])
+#     #             assert len(baseline_model_outputs) == len(preds)
+#     #             logger.info("Baseline model outputs loaded successfully.")
+#     #             judge_results = run_judge(questions,preds,judge_config,baseline_model_outputs,images_bytes)
+#     #         except Exception as e:
+#     #             logger.error(f"Failed while loading model outputs or running judge.")
+#     #             raise e
             
-        elif judge_config["judge_prompt_type"] == "direct assessment":
-            preds = [result["prediction"] for result in results]
-            judge_results = run_judge(questions,preds,judge_config,baseline_model_outputs=None,images=images_bytes)
-        else:
-            logger.error(f"Invalid judge prompt type: {judge_config['judge_prompt_type']}.Skipping judge...")
-            judge_results = None
+#     #     elif judge_config["judge_prompt_type"] == "direct assessment":
+#     #         preds = [result["prediction"] for result in results]
+#     #         judge_results = run_judge(questions,preds,judge_config,baseline_model_outputs=None,images=images_bytes)
+#     #     else:
+#     #         logger.error(f"Invalid judge prompt type: {judge_config['judge_prompt_type']}.Skipping judge...")
+#     #         judge_results = None
         
-    else:
-        judge_results = None
-        logger.info("Judge is disabled, skipping judge...")
-
-    if judge_results is None:
-        return {"judge_results": None}
-    else:
-        results = compute_results(judge_results,judge_config)
-        return results
+#     # else:
+#     #     judge_results = None
+#     #     logger.info("Judge is disabled, skipping judge...")
+
+#     # if judge_results is None:
+#     #     return {"judge_results": None}
+#     # else:
+#     #     results = compute_results(judge_results,judge_config)
+#     #     return results
+#     return {"judge_results": None}
 

From 85065f2a2f23eb561985fff79e76443d7b01914e Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Sun, 27 Apr 2025 14:24:35 +0000
Subject: [PATCH 089/112] script to run eval with llms as judge for aya

---
 run_judge_aya.sh | 103 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 103 insertions(+)
 create mode 100644 run_judge_aya.sh

diff --git a/run_judge_aya.sh b/run_judge_aya.sh
new file mode 100644
index 000000000..23b461c53
--- /dev/null
+++ b/run_judge_aya.sh
@@ -0,0 +1,103 @@
+#!/bin/bash
+
+# -------------------------------
+# Activate the Python environment
+# -------------------------------
+env_path=/mnt/data-poseidon/manos/tower-vision-eval/deepspin-lmms-eval/venv-lmms-eval
+source $env_path/bin/activate
+
+# -------------------------------
+# Export the API key for OpenAI
+# -------------------------------
+
+# -------------------------------
+# Judge and API parameters
+# -------------------------------
+API_TYPE="openai"
+JUDGE_MODEL_NAME="gpt-4o-mini"
+JUDGE_PROMPT_TYPE="comparative"
+TEXT_ONLY="False"
+MAX_TOKENS=2048
+TEMPERATURE=0.0
+TOP_P=1.0
+TENSOR_PARALLEL_SIZE=1
+
+# -------------------------------
+# Define language pairs and models
+# -------------------------------
+lps=("eng")  # List of language codes
+models=("Qwen/Qwen2.5-VL-7B-Instruct" )  # List of model names
+
+# Baseline model name (must match directory structure)
+BASELINE_MODEL_NAME="Unbabel__lnext-qwen2p5-7b-siglip2-v5"
+
+# Root directory for all outputs
+OUT_ROOT="/mnt/scratch-artemis/manos/data/tower-vision-eval-outputs/tower-vision/tower-vision-debug-results"
+
+# -------------------------------
+# Main loop over language pairs
+# -------------------------------
+for lp in "${lps[@]}"; do
+    # ------------------------------------------
+    # Find the most recent baseline output file
+    # ------------------------------------------
+    baseline_dir="$OUT_ROOT/aya-vision-bench-gen-${lp}/${BASELINE_MODEL_NAME}"
+    BASELINE_OUTPUT_PATH=$(ls -1t "$baseline_dir"/*samples_aya-vision-bench-gen-${lp}.jsonl 2>/dev/null | head -n 1)
+    if [ -z "$BASELINE_OUTPUT_PATH" ]; then
+        echo "No baseline output file found for $BASELINE_MODEL_NAME and $lp in $baseline_dir, skipping language."
+        continue
+    fi
+
+    # ------------------------------------------
+    # Loop over models for this language
+    # ------------------------------------------
+    for model in "${models[@]}"; do
+        # Convert model name to directory-friendly format
+        model_dir="${model//\//__}"
+
+        # ------------------------------------------
+        # Find the most recent model output file
+        # ------------------------------------------
+        model_output_dir="$OUT_ROOT/aya-vision-bench-gen-${lp}/${model_dir}"
+        MODEL_OUTPUT_PATH=$(ls -1t "$model_output_dir"/*samples_aya-vision-bench-gen-${lp}.jsonl 2>/dev/null | head -n 1)
+        if [ -z "$MODEL_OUTPUT_PATH" ]; then
+            echo "No model output file found for $model ($model_dir) and $lp in $model_output_dir, skipping."
+            continue
+        fi
+
+        # ------------------------------------------
+        # Set output directory for judge results
+        # ------------------------------------------
+        # Extract the version (filename without extension) from the model output path
+        MODEL_OUTPUT_FILENAME=$(basename "$MODEL_OUTPUT_PATH")
+        MODEL_OUTPUT_VERSION="${MODEL_OUTPUT_FILENAME%.jsonl}"
+        MODEL_OUTPUT_VERSION_CLEAN="${MODEL_OUTPUT_VERSION%_samples_aya-vision-bench-gen-${lp}}"
+
+        # Create the results directory using the version
+        RESULTS_DIR="$OUT_ROOT/aya-vision-bench-gen-${lp}/${model_dir}/${MODEL_OUTPUT_VERSION_CLEAN}_${model_dir}_vs_${BASELINE_MODEL_NAME}_with_${JUDGE_MODEL_NAME}"
+        mkdir -p "$RESULTS_DIR" 
+
+        judge_script_path=/mnt/data-poseidon/manos/tower-vision-eval/deepspin-lmms-eval/lmms-eval/lmms_eval/tasks/ayavisionbench/judge_eval.py
+        # ------------------------------------------
+        # Run the judge evaluation script
+        # ------------------------------------------
+        echo "Running for model: $model, language: $lp" 
+
+        python $judge_script_path \
+            --api_type $API_TYPE \
+            --judge_model_name $JUDGE_MODEL_NAME \
+            --judge_prompt_type $JUDGE_PROMPT_TYPE \
+            --text_only $TEXT_ONLY \
+            --model_output_path "$MODEL_OUTPUT_PATH" \
+            --baseline_output_path "$BASELINE_OUTPUT_PATH" \
+            --lp $lp \
+            --output_dir $RESULTS_DIR \
+            --save_judge_parsed_outputs True \
+            --save_judge_full_responses True \
+            --save_judge_results True \
+            --max_tokens $MAX_TOKENS \
+            --temperature $TEMPERATURE \
+            --top_p $TOP_P \
+            --tensor_parallel_size $TENSOR_PARALLEL_SIZE
+    done
+done
\ No newline at end of file

From af0ae6e6e3464e09ba1c9cab49ee77d08ab2bf5e Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Sun, 27 Apr 2025 14:25:39 +0000
Subject: [PATCH 090/112] small update

---
 lmms_eval/tasks/ayavisionbench/judge_eval.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lmms_eval/tasks/ayavisionbench/judge_eval.py b/lmms_eval/tasks/ayavisionbench/judge_eval.py
index 4b9860a82..1d19701f7 100644
--- a/lmms_eval/tasks/ayavisionbench/judge_eval.py
+++ b/lmms_eval/tasks/ayavisionbench/judge_eval.py
@@ -53,7 +53,7 @@ def load_aya_vision_bench_data(language):
     }
     dataset = load_dataset("CohereForAI/AyaVisionBench",name=mapping_language_to_dataset_name[language],split="test")
     logger.info(f"Loaded Aya Vision Bench dataset for language: {language}")
-    dataset = dataset.select(range(5))
+    # dataset = dataset.select(range(5))
     # Extract questions and images from the dataset
     questions = [item["question"] for item in dataset]
 

From f6fd1f533f5c37dc1297fed4f1a0020056e2d01b Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Sun, 27 Apr 2025 14:46:20 +0000
Subject: [PATCH 091/112] small updates aya

---
 .../ayavisionbench/eval_judge_template.yaml   |  2 +-
 lmms_eval/tasks/ayavisionbench/judge_utils.py | 20 +++++++++----------
 lmms_eval/tasks/ayavisionbench/utils.py       | 10 +++++-----
 3 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/lmms_eval/tasks/ayavisionbench/eval_judge_template.yaml b/lmms_eval/tasks/ayavisionbench/eval_judge_template.yaml
index fcf0e0a8a..8a657e06c 100644
--- a/lmms_eval/tasks/ayavisionbench/eval_judge_template.yaml
+++ b/lmms_eval/tasks/ayavisionbench/eval_judge_template.yaml
@@ -7,7 +7,7 @@
 # judge_prompt_type: comparative
 # text_only: false
 # baseline_model_outputs_path: /mnt/scratch-artemis/manos/data/tower-vision-eval-outputs/tower-vision/tower-vision-debug-results/aya-vision-bench-gen-eng/Unbabel__lnext-qwen2p5-7b-siglip2-v5/Unbabel__lnext-qwen2p5-7b-siglip2-v5/20250418_015501_samples_aya-vision-bench-gen-eng.jsonl #/mnt/scratch-artemis/manos/data/tower-vision-eval-outputs/tower-vision/tower-vision-debug-results/aya-vision-bench-gen-eng/Unbabel__lnext-qwen2p5-7b-siglip2-v5/Unbabel__lnext-qwen2p5-7b-siglip2-v5/20250416_060306_samples_aya-vision-bench-gen-eng.jsonl
-# # max_tokens: 2048
+max_tokens: 2048
 # temperature: 0.0
 # top_p: 1.0
 # tensor_parallel_size: 1
diff --git a/lmms_eval/tasks/ayavisionbench/judge_utils.py b/lmms_eval/tasks/ayavisionbench/judge_utils.py
index ea0a64a5b..7b07d1567 100644
--- a/lmms_eval/tasks/ayavisionbench/judge_utils.py
+++ b/lmms_eval/tasks/ayavisionbench/judge_utils.py
@@ -57,16 +57,16 @@ def img_bytes_to_url(image_dict: dict) -> str:
         logger.error(f"Error converting image bytes to URL: {str(e)}")
         return None
 
-def get_judge_config():
-    with open(Path(__file__).parent / "eval_judge_template.yaml", "r") as f:
-        raw_data = f.readlines()
-        safe_data = []
-        for i, line in enumerate(raw_data):
-            # remove function definition since yaml load cannot handle it
-            if "!function" not in line:
-                safe_data.append(line)
-        config = yaml.safe_load("".join(safe_data))
-    return config
+# def get_judge_config():
+#     with open(Path(__file__).parent / "eval_judge_template.yaml", "r") as f:
+#         raw_data = f.readlines()
+#         safe_data = []
+#         for i, line in enumerate(raw_data):
+#             # remove function definition since yaml load cannot handle it
+#             if "!function" not in line:
+#                 safe_data.append(line)
+#         config = yaml.safe_load("".join(safe_data))
+#     return config
 
 
 def set_prompts(judge_config,questions,preds,baseline_model_outputs=None):
diff --git a/lmms_eval/tasks/ayavisionbench/utils.py b/lmms_eval/tasks/ayavisionbench/utils.py
index 060adbd0d..fe1dba446 100644
--- a/lmms_eval/tasks/ayavisionbench/utils.py
+++ b/lmms_eval/tasks/ayavisionbench/utils.py
@@ -7,11 +7,11 @@
 import io
 from copy import deepcopy
 
-from lmms_eval.tasks.ayavisionbench.judge_utils import (
-    get_judge_config,
-    run_judge,
-    compute_results
-)
+# from lmms_eval.tasks.ayavisionbench.judge_utils import (
+#     get_judge_config,
+#     run_judge,
+#     compute_results
+# )
 
 def pil_to_image_dict(pil_img):
     """

From 38e94531563d0a5c7efcffed37f1d7b2102927de Mon Sep 17 00:00:00 2001
From: Sonal Sannigrahi <sannigrahisonal@gmail.com>
Date: Mon, 12 May 2025 15:06:04 +0100
Subject: [PATCH 092/112] final_answer parser

---
 lmms_eval/utils.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/lmms_eval/utils.py b/lmms_eval/utils.py
index 1604f9a1f..e50b0aef9 100755
--- a/lmms_eval/utils.py
+++ b/lmms_eval/utils.py
@@ -79,6 +79,13 @@ def escaped_split(text, sep_char, maxsplit=-1):
 
     return re.split(r"(?<!\\)" + sep_char, text, maxsplit)
 
+## final answer string exraction when using this parsing format
+def extract_final_answer(text):
+    match = re.search(r'Final Answer:\s*(.*)', text)
+    if match:
+        return match.group(1)
+    else:
+        return text
 
 def handle_arg_string(arg):
     if arg.lower() == "true":

From 91439f27e1063f0c422d80f4de93513df6874893 Mon Sep 17 00:00:00 2001
From: Sonal Sannigrahi <sannigrahisonal@gmail.com>
Date: Mon, 12 May 2025 15:06:51 +0100
Subject: [PATCH 093/112] final_answer parser

---
 lmms_eval/tasks/alm_bench/utils.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/lmms_eval/tasks/alm_bench/utils.py b/lmms_eval/tasks/alm_bench/utils.py
index 9e1f31cbb..0d0a8adf3 100644
--- a/lmms_eval/tasks/alm_bench/utils.py
+++ b/lmms_eval/tasks/alm_bench/utils.py
@@ -2,6 +2,7 @@
 import re
 import sys
 import numpy as np 
+from lmms_eval.utils import extract_final_answer
 
 def exact_match(pred, target):
     if pred == target:
@@ -20,14 +21,14 @@ def split_answer_options(text):
         "korean": "옵션",
         "Chinese (Simplified)": "选项",
         "Spanish": "Opciones",
-        "Italian": "opzioni",
+        "Italian": "Opzioni",
         "Russian": "Варианты",
-        "French": "choix",
+        "French": "choix ",
         "Portuguese": "Opções",
         "German": "Optionen",
     }
     text = text.strip()
-    match = re.match(r"^(.*?)\s*\((?:Options|Opties|옵션|选项|Opciones|opzioni|Варианты|choix |Opções|Optionen):\s*(.*?)\)$", text, re.IGNORECASE)
+    match = re.match(r"^(.*?)\s*\((?:Options|Opties|옵션|선택|선택사항|선택 |选项|Opciones|opzioni|Варианты|choix |Opções|Optionen|Zutaten|Auswahl):\s*(.*?)\)$", text, re.IGNORECASE)
     if match:
         true_answer = match.group(1).strip()
         choices = re.sub("\s*,\s*", "\n", match.group(2))
@@ -45,10 +46,12 @@ def alm_bench_doc_to_text(doc, lmms_eval_specific_kwargs):
 
 
 def alm_bench_process_results(doc, results):
-    pred = results[0]
+    pred = extract_final_answer(results[0])
     target, _ = split_answer_options(doc["Translated_Answer"])
-    match = exact_match(pred, target)
-    return {"results": match}
+    if target == None:
+        print(doc["Translated_Answer"])
+    match = exact_match(pred, target.strip("."))
+    return {"match": match}
 
 
 def alm_bench_doc_to_target(doc, model_specific_target_kwargs):

From a62ad3a8af249f53185ae1e60b1adac589c812bc Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel.com>
Date: Wed, 14 May 2025 09:43:05 +0000
Subject: [PATCH 094/112] fixed the bugs of alm-bench

---
 lmms_eval/models/llava.py                     |   9 +-
 .../tasks/alm_bench/_default_template_yaml    |  29 ++
 lmms_eval/tasks/alm_bench/alm-bench-de.yaml   |   8 +-
 lmms_eval/tasks/alm_bench/alm-bench-en.yaml   |   6 +-
 lmms_eval/tasks/alm_bench/alm-bench-es.yaml   |   8 +-
 lmms_eval/tasks/alm_bench/alm-bench-fr.yaml   |   8 +-
 lmms_eval/tasks/alm_bench/alm-bench-it.yaml   |   8 +-
 lmms_eval/tasks/alm_bench/alm-bench-ko.yaml   |   8 +-
 lmms_eval/tasks/alm_bench/alm-bench-nl.yaml   |   8 +-
 lmms_eval/tasks/alm_bench/alm-bench-pt.yaml   |   8 +-
 lmms_eval/tasks/alm_bench/alm-bench-ru.yaml   |   8 +-
 lmms_eval/tasks/alm_bench/utils.py            |   2 +-
 .../aya-vision-bench-gen-cs.yaml              |  34 ---
 .../aya-vision-bench-gen-de.yaml              |  34 ---
 .../aya-vision-bench-gen-eng.yaml             |  35 ---
 .../aya-vision-bench-gen-es.yaml              |  34 ---
 .../aya-vision-bench-gen-fr.yaml              |  35 ---
 .../aya-vision-bench-gen-it.yaml              |  35 ---
 .../aya-vision-bench-gen-ko.yaml              |  35 ---
 .../aya-vision-bench-gen-nl.yaml              |  35 ---
 .../aya-vision-bench-gen-pt.yaml              |  35 ---
 .../aya-vision-bench-gen-ru.yaml              |  35 ---
 .../aya-vision-bench-gen-zh.yaml              |  35 ---
 .../ayavisionbench/aya-vision-bench-gen.yaml  |  14 -
 .../ayavisionbench/eval_judge_template.yaml   |  16 --
 lmms_eval/tasks/ayavisionbench/judge_eval.py  | 151 -----------
 .../tasks/ayavisionbench/judge_templates.py   |  39 ---
 lmms_eval/tasks/ayavisionbench/judge_utils.py | 255 ------------------
 lmms_eval/tasks/ayavisionbench/utils.py       | 131 ---------
 lmms_eval/tasks/global_mmlu/README.md         |  42 +++
 .../global_mmlu/default/ar/_ar_template_yaml  |  16 ++
 .../default/ar/_global_mmlu_ar.yaml           |  13 +
 .../default/ar/global_mmlu_ar_business.yaml   |   4 +
 .../default/ar/global_mmlu_ar_humanities.yaml |   4 +
 .../default/ar/global_mmlu_ar_medical.yaml    |   4 +
 .../default/ar/global_mmlu_ar_other.yaml      |   4 +
 .../ar/global_mmlu_ar_social_sciences.yaml    |   4 +
 .../default/ar/global_mmlu_ar_stem.yaml       |   4 +
 .../tasks/global_mmlu/default/ar/utils.py     |  18 ++
 .../global_mmlu/default/bn/_bn_template_yaml  |  16 ++
 .../default/bn/_global_mmlu_bn.yaml           |  13 +
 .../default/bn/global_mmlu_bn_business.yaml   |   4 +
 .../default/bn/global_mmlu_bn_humanities.yaml |   4 +
 .../default/bn/global_mmlu_bn_medical.yaml    |   4 +
 .../default/bn/global_mmlu_bn_other.yaml      |   4 +
 .../bn/global_mmlu_bn_social_sciences.yaml    |   4 +
 .../default/bn/global_mmlu_bn_stem.yaml       |   4 +
 .../tasks/global_mmlu/default/bn/utils.py     |  18 ++
 .../global_mmlu/default/de/_de_template_yaml  |  16 ++
 .../default/de/_global_mmlu_de.yaml           |  13 +
 .../default/de/global_mmlu_de_business.yaml   |   4 +
 .../default/de/global_mmlu_de_humanities.yaml |   4 +
 .../default/de/global_mmlu_de_medical.yaml    |   4 +
 .../default/de/global_mmlu_de_other.yaml      |   4 +
 .../de/global_mmlu_de_social_sciences.yaml    |   4 +
 .../default/de/global_mmlu_de_stem.yaml       |   4 +
 .../tasks/global_mmlu/default/de/utils.py     |  18 ++
 .../global_mmlu/default/en/_en_template_yaml  |  16 ++
 .../default/en/_global_mmlu_en.yaml           |  13 +
 .../default/en/global_mmlu_en_business.yaml   |   4 +
 .../default/en/global_mmlu_en_humanities.yaml |   4 +
 .../default/en/global_mmlu_en_medical.yaml    |   4 +
 .../default/en/global_mmlu_en_other.yaml      |   4 +
 .../en/global_mmlu_en_social_sciences.yaml    |   4 +
 .../default/en/global_mmlu_en_stem.yaml       |   4 +
 .../tasks/global_mmlu/default/en/utils.py     |  18 ++
 .../global_mmlu/default/es/_es_template_yaml  |  16 ++
 .../default/es/_global_mmlu_es.yaml           |  13 +
 .../default/es/global_mmlu_es_business.yaml   |   4 +
 .../default/es/global_mmlu_es_humanities.yaml |   4 +
 .../default/es/global_mmlu_es_medical.yaml    |   4 +
 .../default/es/global_mmlu_es_other.yaml      |   4 +
 .../es/global_mmlu_es_social_sciences.yaml    |   4 +
 .../default/es/global_mmlu_es_stem.yaml       |   4 +
 .../tasks/global_mmlu/default/es/utils.py     |  18 ++
 .../global_mmlu/default/fr/_fr_template_yaml  |  16 ++
 .../default/fr/_global_mmlu_fr.yaml           |  13 +
 .../default/fr/global_mmlu_fr_business.yaml   |   4 +
 .../default/fr/global_mmlu_fr_humanities.yaml |   4 +
 .../default/fr/global_mmlu_fr_medical.yaml    |   4 +
 .../default/fr/global_mmlu_fr_other.yaml      |   4 +
 .../fr/global_mmlu_fr_social_sciences.yaml    |   4 +
 .../default/fr/global_mmlu_fr_stem.yaml       |   4 +
 .../tasks/global_mmlu/default/fr/utils.py     |  18 ++
 .../default/hi/_global_mmlu_hi.yaml           |  13 +
 .../global_mmlu/default/hi/_hi_template_yaml  |  16 ++
 .../default/hi/global_mmlu_hi_business.yaml   |   4 +
 .../default/hi/global_mmlu_hi_humanities.yaml |   4 +
 .../default/hi/global_mmlu_hi_medical.yaml    |   4 +
 .../default/hi/global_mmlu_hi_other.yaml      |   4 +
 .../hi/global_mmlu_hi_social_sciences.yaml    |   4 +
 .../default/hi/global_mmlu_hi_stem.yaml       |   4 +
 .../tasks/global_mmlu/default/hi/utils.py     |  18 ++
 .../default/id/_global_mmlu_id.yaml           |  13 +
 .../global_mmlu/default/id/_id_template_yaml  |  16 ++
 .../default/id/global_mmlu_id_business.yaml   |   4 +
 .../default/id/global_mmlu_id_humanities.yaml |   4 +
 .../default/id/global_mmlu_id_medical.yaml    |   4 +
 .../default/id/global_mmlu_id_other.yaml      |   4 +
 .../id/global_mmlu_id_social_sciences.yaml    |   4 +
 .../default/id/global_mmlu_id_stem.yaml       |   4 +
 .../tasks/global_mmlu/default/id/utils.py     |  18 ++
 .../default/it/_global_mmlu_it.yaml           |  13 +
 .../global_mmlu/default/it/_it_template_yaml  |  16 ++
 .../default/it/global_mmlu_it_business.yaml   |   4 +
 .../default/it/global_mmlu_it_humanities.yaml |   4 +
 .../default/it/global_mmlu_it_medical.yaml    |   4 +
 .../default/it/global_mmlu_it_other.yaml      |   4 +
 .../it/global_mmlu_it_social_sciences.yaml    |   4 +
 .../default/it/global_mmlu_it_stem.yaml       |   4 +
 .../tasks/global_mmlu/default/it/utils.py     |  18 ++
 .../default/ja/_global_mmlu_ja.yaml           |  13 +
 .../global_mmlu/default/ja/_ja_template_yaml  |  16 ++
 .../default/ja/global_mmlu_ja_business.yaml   |   4 +
 .../default/ja/global_mmlu_ja_humanities.yaml |   4 +
 .../default/ja/global_mmlu_ja_medical.yaml    |   4 +
 .../default/ja/global_mmlu_ja_other.yaml      |   4 +
 .../ja/global_mmlu_ja_social_sciences.yaml    |   4 +
 .../default/ja/global_mmlu_ja_stem.yaml       |   4 +
 .../tasks/global_mmlu/default/ja/utils.py     |  18 ++
 .../default/ko/_global_mmlu_ko.yaml           |  13 +
 .../global_mmlu/default/ko/_ko_template_yaml  |  16 ++
 .../default/ko/global_mmlu_ko_business.yaml   |   4 +
 .../default/ko/global_mmlu_ko_humanities.yaml |   4 +
 .../default/ko/global_mmlu_ko_medical.yaml    |   4 +
 .../default/ko/global_mmlu_ko_other.yaml      |   4 +
 .../ko/global_mmlu_ko_social_sciences.yaml    |   4 +
 .../default/ko/global_mmlu_ko_stem.yaml       |   4 +
 .../tasks/global_mmlu/default/ko/utils.py     |  18 ++
 .../default/pt/_global_mmlu_pt.yaml           |  13 +
 .../global_mmlu/default/pt/_pt_template_yaml  |  16 ++
 .../default/pt/global_mmlu_pt_business.yaml   |   4 +
 .../default/pt/global_mmlu_pt_humanities.yaml |   4 +
 .../default/pt/global_mmlu_pt_medical.yaml    |   4 +
 .../default/pt/global_mmlu_pt_other.yaml      |   4 +
 .../pt/global_mmlu_pt_social_sciences.yaml    |   4 +
 .../default/pt/global_mmlu_pt_stem.yaml       |   4 +
 .../tasks/global_mmlu/default/pt/utils.py     |  18 ++
 .../default/sw/_global_mmlu_sw.yaml           |  13 +
 .../global_mmlu/default/sw/_sw_template_yaml  |  16 ++
 .../default/sw/global_mmlu_sw_business.yaml   |   4 +
 .../default/sw/global_mmlu_sw_humanities.yaml |   4 +
 .../default/sw/global_mmlu_sw_medical.yaml    |   4 +
 .../default/sw/global_mmlu_sw_other.yaml      |   4 +
 .../sw/global_mmlu_sw_social_sciences.yaml    |   4 +
 .../default/sw/global_mmlu_sw_stem.yaml       |   4 +
 .../tasks/global_mmlu/default/sw/utils.py     |  18 ++
 .../default/yo/_global_mmlu_yo.yaml           |  13 +
 .../global_mmlu/default/yo/_yo_template_yaml  |  16 ++
 .../default/yo/global_mmlu_yo_business.yaml   |   4 +
 .../default/yo/global_mmlu_yo_humanities.yaml |   4 +
 .../default/yo/global_mmlu_yo_medical.yaml    |   4 +
 .../default/yo/global_mmlu_yo_other.yaml      |   4 +
 .../yo/global_mmlu_yo_social_sciences.yaml    |   4 +
 .../default/yo/global_mmlu_yo_stem.yaml       |   4 +
 .../tasks/global_mmlu/default/yo/utils.py     |  18 ++
 .../default/zh/_global_mmlu_zh.yaml           |  13 +
 .../global_mmlu/default/zh/_zh_template_yaml  |  16 ++
 .../default/zh/global_mmlu_zh_business.yaml   |   4 +
 .../default/zh/global_mmlu_zh_humanities.yaml |   4 +
 .../default/zh/global_mmlu_zh_medical.yaml    |   4 +
 .../default/zh/global_mmlu_zh_other.yaml      |   4 +
 .../zh/global_mmlu_zh_social_sciences.yaml    |   4 +
 .../default/zh/global_mmlu_zh_stem.yaml       |   4 +
 .../tasks/global_mmlu/default/zh/utils.py     |  18 ++
 .../global_mmlu/full/am/_am_template_yaml     |  16 ++
 .../full/am/_global_mmlu_full_am.yaml         |  11 +
 .../am/_global_mmlu_full_am_humanities.yaml   |   8 +
 .../full/am/_global_mmlu_full_am_other.yaml   |   8 +
 .../_global_mmlu_full_am_social_sciences.yaml |   8 +
 .../full/am/_global_mmlu_full_am_stem.yaml    |   8 +
 .../global_mmlu_full_am_abstract_algebra.yaml |   5 +
 .../full/am/global_mmlu_full_am_anatomy.yaml  |   5 +
 .../am/global_mmlu_full_am_astronomy.yaml     |   5 +
 .../global_mmlu_full_am_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_am_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_am_college_biology.yaml  |   5 +
 ...global_mmlu_full_am_college_chemistry.yaml |   5 +
 ...mmlu_full_am_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_am_college_mathematics.yaml |   5 +
 .../global_mmlu_full_am_college_medicine.yaml |   5 +
 .../global_mmlu_full_am_college_physics.yaml  |   5 +
 ...global_mmlu_full_am_computer_security.yaml |   5 +
 ...lobal_mmlu_full_am_conceptual_physics.yaml |   5 +
 .../am/global_mmlu_full_am_econometrics.yaml  |   5 +
 ...l_mmlu_full_am_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_am_elementary_mathematics.yaml |   5 +
 .../am/global_mmlu_full_am_formal_logic.yaml  |   5 +
 .../am/global_mmlu_full_am_global_facts.yaml  |   5 +
 ...obal_mmlu_full_am_high_school_biology.yaml |   5 +
 ...al_mmlu_full_am_high_school_chemistry.yaml |   5 +
 ..._full_am_high_school_computer_science.yaml |   5 +
 ..._full_am_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_am_high_school_geography.yaml |   5 +
 ...m_high_school_government_and_politics.yaml |   5 +
 ...lu_full_am_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_am_high_school_mathematics.yaml |   5 +
 ...lu_full_am_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_am_high_school_physics.yaml |   5 +
 ...l_mmlu_full_am_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_am_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_am_high_school_us_history.yaml |   5 +
 ...mlu_full_am_high_school_world_history.yaml |   5 +
 .../am/global_mmlu_full_am_human_aging.yaml   |   5 +
 .../global_mmlu_full_am_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_am_international_law.yaml |   5 +
 .../am/global_mmlu_full_am_jurisprudence.yaml |   5 +
 ...global_mmlu_full_am_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_am_machine_learning.yaml |   5 +
 .../am/global_mmlu_full_am_management.yaml    |   5 +
 .../am/global_mmlu_full_am_marketing.yaml     |   5 +
 .../global_mmlu_full_am_medical_genetics.yaml |   5 +
 .../am/global_mmlu_full_am_miscellaneous.yaml |   5 +
 .../global_mmlu_full_am_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_am_moral_scenarios.yaml  |   5 +
 .../am/global_mmlu_full_am_nutrition.yaml     |   5 +
 .../am/global_mmlu_full_am_philosophy.yaml    |   5 +
 .../am/global_mmlu_full_am_prehistory.yaml    |   5 +
 ..._mmlu_full_am_professional_accounting.yaml |   5 +
 .../global_mmlu_full_am_professional_law.yaml |   5 +
 ...al_mmlu_full_am_professional_medicine.yaml |   5 +
 ..._mmlu_full_am_professional_psychology.yaml |   5 +
 .../global_mmlu_full_am_public_relations.yaml |   5 +
 .../global_mmlu_full_am_security_studies.yaml |   5 +
 .../am/global_mmlu_full_am_sociology.yaml     |   5 +
 ...global_mmlu_full_am_us_foreign_policy.yaml |   5 +
 .../full/am/global_mmlu_full_am_virology.yaml |   5 +
 .../global_mmlu_full_am_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/am/utils.py  |  73 +++++
 .../global_mmlu/full/ar/_ar_template_yaml     |  16 ++
 .../full/ar/_global_mmlu_full_ar.yaml         |  11 +
 .../ar/_global_mmlu_full_ar_humanities.yaml   |   8 +
 .../full/ar/_global_mmlu_full_ar_other.yaml   |   8 +
 .../_global_mmlu_full_ar_social_sciences.yaml |   8 +
 .../full/ar/_global_mmlu_full_ar_stem.yaml    |   8 +
 .../global_mmlu_full_ar_abstract_algebra.yaml |   5 +
 .../full/ar/global_mmlu_full_ar_anatomy.yaml  |   5 +
 .../ar/global_mmlu_full_ar_astronomy.yaml     |   5 +
 .../global_mmlu_full_ar_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_ar_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_ar_college_biology.yaml  |   5 +
 ...global_mmlu_full_ar_college_chemistry.yaml |   5 +
 ...mmlu_full_ar_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_ar_college_mathematics.yaml |   5 +
 .../global_mmlu_full_ar_college_medicine.yaml |   5 +
 .../global_mmlu_full_ar_college_physics.yaml  |   5 +
 ...global_mmlu_full_ar_computer_security.yaml |   5 +
 ...lobal_mmlu_full_ar_conceptual_physics.yaml |   5 +
 .../ar/global_mmlu_full_ar_econometrics.yaml  |   5 +
 ...l_mmlu_full_ar_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_ar_elementary_mathematics.yaml |   5 +
 .../ar/global_mmlu_full_ar_formal_logic.yaml  |   5 +
 .../ar/global_mmlu_full_ar_global_facts.yaml  |   5 +
 ...obal_mmlu_full_ar_high_school_biology.yaml |   5 +
 ...al_mmlu_full_ar_high_school_chemistry.yaml |   5 +
 ..._full_ar_high_school_computer_science.yaml |   5 +
 ..._full_ar_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_ar_high_school_geography.yaml |   5 +
 ...r_high_school_government_and_politics.yaml |   5 +
 ...lu_full_ar_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_ar_high_school_mathematics.yaml |   5 +
 ...lu_full_ar_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_ar_high_school_physics.yaml |   5 +
 ...l_mmlu_full_ar_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_ar_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_ar_high_school_us_history.yaml |   5 +
 ...mlu_full_ar_high_school_world_history.yaml |   5 +
 .../ar/global_mmlu_full_ar_human_aging.yaml   |   5 +
 .../global_mmlu_full_ar_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_ar_international_law.yaml |   5 +
 .../ar/global_mmlu_full_ar_jurisprudence.yaml |   5 +
 ...global_mmlu_full_ar_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_ar_machine_learning.yaml |   5 +
 .../ar/global_mmlu_full_ar_management.yaml    |   5 +
 .../ar/global_mmlu_full_ar_marketing.yaml     |   5 +
 .../global_mmlu_full_ar_medical_genetics.yaml |   5 +
 .../ar/global_mmlu_full_ar_miscellaneous.yaml |   5 +
 .../global_mmlu_full_ar_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_ar_moral_scenarios.yaml  |   5 +
 .../ar/global_mmlu_full_ar_nutrition.yaml     |   5 +
 .../ar/global_mmlu_full_ar_philosophy.yaml    |   5 +
 .../ar/global_mmlu_full_ar_prehistory.yaml    |   5 +
 ..._mmlu_full_ar_professional_accounting.yaml |   5 +
 .../global_mmlu_full_ar_professional_law.yaml |   5 +
 ...al_mmlu_full_ar_professional_medicine.yaml |   5 +
 ..._mmlu_full_ar_professional_psychology.yaml |   5 +
 .../global_mmlu_full_ar_public_relations.yaml |   5 +
 .../global_mmlu_full_ar_security_studies.yaml |   5 +
 .../ar/global_mmlu_full_ar_sociology.yaml     |   5 +
 ...global_mmlu_full_ar_us_foreign_policy.yaml |   5 +
 .../full/ar/global_mmlu_full_ar_virology.yaml |   5 +
 .../global_mmlu_full_ar_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/ar/utils.py  |  73 +++++
 .../global_mmlu/full/bn/_bn_template_yaml     |  16 ++
 .../full/bn/_global_mmlu_full_bn.yaml         |  11 +
 .../bn/_global_mmlu_full_bn_humanities.yaml   |   8 +
 .../full/bn/_global_mmlu_full_bn_other.yaml   |   8 +
 .../_global_mmlu_full_bn_social_sciences.yaml |   8 +
 .../full/bn/_global_mmlu_full_bn_stem.yaml    |   8 +
 .../global_mmlu_full_bn_abstract_algebra.yaml |   5 +
 .../full/bn/global_mmlu_full_bn_anatomy.yaml  |   5 +
 .../bn/global_mmlu_full_bn_astronomy.yaml     |   5 +
 .../global_mmlu_full_bn_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_bn_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_bn_college_biology.yaml  |   5 +
 ...global_mmlu_full_bn_college_chemistry.yaml |   5 +
 ...mmlu_full_bn_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_bn_college_mathematics.yaml |   5 +
 .../global_mmlu_full_bn_college_medicine.yaml |   5 +
 .../global_mmlu_full_bn_college_physics.yaml  |   5 +
 ...global_mmlu_full_bn_computer_security.yaml |   5 +
 ...lobal_mmlu_full_bn_conceptual_physics.yaml |   5 +
 .../bn/global_mmlu_full_bn_econometrics.yaml  |   5 +
 ...l_mmlu_full_bn_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_bn_elementary_mathematics.yaml |   5 +
 .../bn/global_mmlu_full_bn_formal_logic.yaml  |   5 +
 .../bn/global_mmlu_full_bn_global_facts.yaml  |   5 +
 ...obal_mmlu_full_bn_high_school_biology.yaml |   5 +
 ...al_mmlu_full_bn_high_school_chemistry.yaml |   5 +
 ..._full_bn_high_school_computer_science.yaml |   5 +
 ..._full_bn_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_bn_high_school_geography.yaml |   5 +
 ...n_high_school_government_and_politics.yaml |   5 +
 ...lu_full_bn_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_bn_high_school_mathematics.yaml |   5 +
 ...lu_full_bn_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_bn_high_school_physics.yaml |   5 +
 ...l_mmlu_full_bn_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_bn_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_bn_high_school_us_history.yaml |   5 +
 ...mlu_full_bn_high_school_world_history.yaml |   5 +
 .../bn/global_mmlu_full_bn_human_aging.yaml   |   5 +
 .../global_mmlu_full_bn_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_bn_international_law.yaml |   5 +
 .../bn/global_mmlu_full_bn_jurisprudence.yaml |   5 +
 ...global_mmlu_full_bn_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_bn_machine_learning.yaml |   5 +
 .../bn/global_mmlu_full_bn_management.yaml    |   5 +
 .../bn/global_mmlu_full_bn_marketing.yaml     |   5 +
 .../global_mmlu_full_bn_medical_genetics.yaml |   5 +
 .../bn/global_mmlu_full_bn_miscellaneous.yaml |   5 +
 .../global_mmlu_full_bn_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_bn_moral_scenarios.yaml  |   5 +
 .../bn/global_mmlu_full_bn_nutrition.yaml     |   5 +
 .../bn/global_mmlu_full_bn_philosophy.yaml    |   5 +
 .../bn/global_mmlu_full_bn_prehistory.yaml    |   5 +
 ..._mmlu_full_bn_professional_accounting.yaml |   5 +
 .../global_mmlu_full_bn_professional_law.yaml |   5 +
 ...al_mmlu_full_bn_professional_medicine.yaml |   5 +
 ..._mmlu_full_bn_professional_psychology.yaml |   5 +
 .../global_mmlu_full_bn_public_relations.yaml |   5 +
 .../global_mmlu_full_bn_security_studies.yaml |   5 +
 .../bn/global_mmlu_full_bn_sociology.yaml     |   5 +
 ...global_mmlu_full_bn_us_foreign_policy.yaml |   5 +
 .../full/bn/global_mmlu_full_bn_virology.yaml |   5 +
 .../global_mmlu_full_bn_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/bn/utils.py  |  73 +++++
 .../global_mmlu/full/cs/_cs_template_yaml     |  16 ++
 .../full/cs/_global_mmlu_full_cs.yaml         |  11 +
 .../cs/_global_mmlu_full_cs_humanities.yaml   |   8 +
 .../full/cs/_global_mmlu_full_cs_other.yaml   |   8 +
 .../_global_mmlu_full_cs_social_sciences.yaml |   8 +
 .../full/cs/_global_mmlu_full_cs_stem.yaml    |   8 +
 .../global_mmlu_full_cs_abstract_algebra.yaml |   5 +
 .../full/cs/global_mmlu_full_cs_anatomy.yaml  |   5 +
 .../cs/global_mmlu_full_cs_astronomy.yaml     |   5 +
 .../global_mmlu_full_cs_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_cs_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_cs_college_biology.yaml  |   5 +
 ...global_mmlu_full_cs_college_chemistry.yaml |   5 +
 ...mmlu_full_cs_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_cs_college_mathematics.yaml |   5 +
 .../global_mmlu_full_cs_college_medicine.yaml |   5 +
 .../global_mmlu_full_cs_college_physics.yaml  |   5 +
 ...global_mmlu_full_cs_computer_security.yaml |   5 +
 ...lobal_mmlu_full_cs_conceptual_physics.yaml |   5 +
 .../cs/global_mmlu_full_cs_econometrics.yaml  |   5 +
 ...l_mmlu_full_cs_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_cs_elementary_mathematics.yaml |   5 +
 .../cs/global_mmlu_full_cs_formal_logic.yaml  |   5 +
 .../cs/global_mmlu_full_cs_global_facts.yaml  |   5 +
 ...obal_mmlu_full_cs_high_school_biology.yaml |   5 +
 ...al_mmlu_full_cs_high_school_chemistry.yaml |   5 +
 ..._full_cs_high_school_computer_science.yaml |   5 +
 ..._full_cs_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_cs_high_school_geography.yaml |   5 +
 ...s_high_school_government_and_politics.yaml |   5 +
 ...lu_full_cs_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_cs_high_school_mathematics.yaml |   5 +
 ...lu_full_cs_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_cs_high_school_physics.yaml |   5 +
 ...l_mmlu_full_cs_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_cs_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_cs_high_school_us_history.yaml |   5 +
 ...mlu_full_cs_high_school_world_history.yaml |   5 +
 .../cs/global_mmlu_full_cs_human_aging.yaml   |   5 +
 .../global_mmlu_full_cs_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_cs_international_law.yaml |   5 +
 .../cs/global_mmlu_full_cs_jurisprudence.yaml |   5 +
 ...global_mmlu_full_cs_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_cs_machine_learning.yaml |   5 +
 .../cs/global_mmlu_full_cs_management.yaml    |   5 +
 .../cs/global_mmlu_full_cs_marketing.yaml     |   5 +
 .../global_mmlu_full_cs_medical_genetics.yaml |   5 +
 .../cs/global_mmlu_full_cs_miscellaneous.yaml |   5 +
 .../global_mmlu_full_cs_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_cs_moral_scenarios.yaml  |   5 +
 .../cs/global_mmlu_full_cs_nutrition.yaml     |   5 +
 .../cs/global_mmlu_full_cs_philosophy.yaml    |   5 +
 .../cs/global_mmlu_full_cs_prehistory.yaml    |   5 +
 ..._mmlu_full_cs_professional_accounting.yaml |   5 +
 .../global_mmlu_full_cs_professional_law.yaml |   5 +
 ...al_mmlu_full_cs_professional_medicine.yaml |   5 +
 ..._mmlu_full_cs_professional_psychology.yaml |   5 +
 .../global_mmlu_full_cs_public_relations.yaml |   5 +
 .../global_mmlu_full_cs_security_studies.yaml |   5 +
 .../cs/global_mmlu_full_cs_sociology.yaml     |   5 +
 ...global_mmlu_full_cs_us_foreign_policy.yaml |   5 +
 .../full/cs/global_mmlu_full_cs_virology.yaml |   5 +
 .../global_mmlu_full_cs_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/cs/utils.py  |  73 +++++
 .../global_mmlu/full/de/_de_template_yaml     |  16 ++
 .../full/de/_global_mmlu_full_de.yaml         |  11 +
 .../de/_global_mmlu_full_de_humanities.yaml   |   8 +
 .../full/de/_global_mmlu_full_de_other.yaml   |   8 +
 .../_global_mmlu_full_de_social_sciences.yaml |   8 +
 .../full/de/_global_mmlu_full_de_stem.yaml    |   8 +
 .../global_mmlu_full_de_abstract_algebra.yaml |   5 +
 .../full/de/global_mmlu_full_de_anatomy.yaml  |   5 +
 .../de/global_mmlu_full_de_astronomy.yaml     |   5 +
 .../global_mmlu_full_de_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_de_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_de_college_biology.yaml  |   5 +
 ...global_mmlu_full_de_college_chemistry.yaml |   5 +
 ...mmlu_full_de_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_de_college_mathematics.yaml |   5 +
 .../global_mmlu_full_de_college_medicine.yaml |   5 +
 .../global_mmlu_full_de_college_physics.yaml  |   5 +
 ...global_mmlu_full_de_computer_security.yaml |   5 +
 ...lobal_mmlu_full_de_conceptual_physics.yaml |   5 +
 .../de/global_mmlu_full_de_econometrics.yaml  |   5 +
 ...l_mmlu_full_de_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_de_elementary_mathematics.yaml |   5 +
 .../de/global_mmlu_full_de_formal_logic.yaml  |   5 +
 .../de/global_mmlu_full_de_global_facts.yaml  |   5 +
 ...obal_mmlu_full_de_high_school_biology.yaml |   5 +
 ...al_mmlu_full_de_high_school_chemistry.yaml |   5 +
 ..._full_de_high_school_computer_science.yaml |   5 +
 ..._full_de_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_de_high_school_geography.yaml |   5 +
 ...e_high_school_government_and_politics.yaml |   5 +
 ...lu_full_de_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_de_high_school_mathematics.yaml |   5 +
 ...lu_full_de_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_de_high_school_physics.yaml |   5 +
 ...l_mmlu_full_de_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_de_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_de_high_school_us_history.yaml |   5 +
 ...mlu_full_de_high_school_world_history.yaml |   5 +
 .../de/global_mmlu_full_de_human_aging.yaml   |   5 +
 .../global_mmlu_full_de_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_de_international_law.yaml |   5 +
 .../de/global_mmlu_full_de_jurisprudence.yaml |   5 +
 ...global_mmlu_full_de_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_de_machine_learning.yaml |   5 +
 .../de/global_mmlu_full_de_management.yaml    |   5 +
 .../de/global_mmlu_full_de_marketing.yaml     |   5 +
 .../global_mmlu_full_de_medical_genetics.yaml |   5 +
 .../de/global_mmlu_full_de_miscellaneous.yaml |   5 +
 .../global_mmlu_full_de_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_de_moral_scenarios.yaml  |   5 +
 .../de/global_mmlu_full_de_nutrition.yaml     |   5 +
 .../de/global_mmlu_full_de_philosophy.yaml    |   5 +
 .../de/global_mmlu_full_de_prehistory.yaml    |   5 +
 ..._mmlu_full_de_professional_accounting.yaml |   5 +
 .../global_mmlu_full_de_professional_law.yaml |   5 +
 ...al_mmlu_full_de_professional_medicine.yaml |   5 +
 ..._mmlu_full_de_professional_psychology.yaml |   5 +
 .../global_mmlu_full_de_public_relations.yaml |   5 +
 .../global_mmlu_full_de_security_studies.yaml |   5 +
 .../de/global_mmlu_full_de_sociology.yaml     |   5 +
 ...global_mmlu_full_de_us_foreign_policy.yaml |   5 +
 .../full/de/global_mmlu_full_de_virology.yaml |   5 +
 .../global_mmlu_full_de_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/de/utils.py  |  73 +++++
 .../global_mmlu/full/el/_el_template_yaml     |  16 ++
 .../full/el/_global_mmlu_full_el.yaml         |  11 +
 .../el/_global_mmlu_full_el_humanities.yaml   |   8 +
 .../full/el/_global_mmlu_full_el_other.yaml   |   8 +
 .../_global_mmlu_full_el_social_sciences.yaml |   8 +
 .../full/el/_global_mmlu_full_el_stem.yaml    |   8 +
 .../global_mmlu_full_el_abstract_algebra.yaml |   5 +
 .../full/el/global_mmlu_full_el_anatomy.yaml  |   5 +
 .../el/global_mmlu_full_el_astronomy.yaml     |   5 +
 .../global_mmlu_full_el_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_el_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_el_college_biology.yaml  |   5 +
 ...global_mmlu_full_el_college_chemistry.yaml |   5 +
 ...mmlu_full_el_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_el_college_mathematics.yaml |   5 +
 .../global_mmlu_full_el_college_medicine.yaml |   5 +
 .../global_mmlu_full_el_college_physics.yaml  |   5 +
 ...global_mmlu_full_el_computer_security.yaml |   5 +
 ...lobal_mmlu_full_el_conceptual_physics.yaml |   5 +
 .../el/global_mmlu_full_el_econometrics.yaml  |   5 +
 ...l_mmlu_full_el_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_el_elementary_mathematics.yaml |   5 +
 .../el/global_mmlu_full_el_formal_logic.yaml  |   5 +
 .../el/global_mmlu_full_el_global_facts.yaml  |   5 +
 ...obal_mmlu_full_el_high_school_biology.yaml |   5 +
 ...al_mmlu_full_el_high_school_chemistry.yaml |   5 +
 ..._full_el_high_school_computer_science.yaml |   5 +
 ..._full_el_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_el_high_school_geography.yaml |   5 +
 ...l_high_school_government_and_politics.yaml |   5 +
 ...lu_full_el_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_el_high_school_mathematics.yaml |   5 +
 ...lu_full_el_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_el_high_school_physics.yaml |   5 +
 ...l_mmlu_full_el_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_el_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_el_high_school_us_history.yaml |   5 +
 ...mlu_full_el_high_school_world_history.yaml |   5 +
 .../el/global_mmlu_full_el_human_aging.yaml   |   5 +
 .../global_mmlu_full_el_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_el_international_law.yaml |   5 +
 .../el/global_mmlu_full_el_jurisprudence.yaml |   5 +
 ...global_mmlu_full_el_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_el_machine_learning.yaml |   5 +
 .../el/global_mmlu_full_el_management.yaml    |   5 +
 .../el/global_mmlu_full_el_marketing.yaml     |   5 +
 .../global_mmlu_full_el_medical_genetics.yaml |   5 +
 .../el/global_mmlu_full_el_miscellaneous.yaml |   5 +
 .../global_mmlu_full_el_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_el_moral_scenarios.yaml  |   5 +
 .../el/global_mmlu_full_el_nutrition.yaml     |   5 +
 .../el/global_mmlu_full_el_philosophy.yaml    |   5 +
 .../el/global_mmlu_full_el_prehistory.yaml    |   5 +
 ..._mmlu_full_el_professional_accounting.yaml |   5 +
 .../global_mmlu_full_el_professional_law.yaml |   5 +
 ...al_mmlu_full_el_professional_medicine.yaml |   5 +
 ..._mmlu_full_el_professional_psychology.yaml |   5 +
 .../global_mmlu_full_el_public_relations.yaml |   5 +
 .../global_mmlu_full_el_security_studies.yaml |   5 +
 .../el/global_mmlu_full_el_sociology.yaml     |   5 +
 ...global_mmlu_full_el_us_foreign_policy.yaml |   5 +
 .../full/el/global_mmlu_full_el_virology.yaml |   5 +
 .../global_mmlu_full_el_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/el/utils.py  |  73 +++++
 .../global_mmlu/full/en/_en_template_yaml     |  25 ++
 .../full/en/_global_mmlu_full_en.yaml         |  12 +
 .../en/_global_mmlu_full_en_humanities.yaml   |   8 +
 .../full/en/_global_mmlu_full_en_other.yaml   |   8 +
 .../_global_mmlu_full_en_social_sciences.yaml |   8 +
 .../full/en/_global_mmlu_full_en_stem.yaml    |   8 +
 .../global_mmlu_full_en_abstract_algebra.yaml |   5 +
 .../full/en/global_mmlu_full_en_anatomy.yaml  |   5 +
 .../en/global_mmlu_full_en_astronomy.yaml     |   5 +
 .../global_mmlu_full_en_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_en_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_en_college_biology.yaml  |   5 +
 ...global_mmlu_full_en_college_chemistry.yaml |   5 +
 ...mmlu_full_en_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_en_college_mathematics.yaml |   5 +
 .../global_mmlu_full_en_college_medicine.yaml |   5 +
 .../global_mmlu_full_en_college_physics.yaml  |   5 +
 ...global_mmlu_full_en_computer_security.yaml |   5 +
 ...lobal_mmlu_full_en_conceptual_physics.yaml |   5 +
 .../en/global_mmlu_full_en_econometrics.yaml  |   5 +
 ...l_mmlu_full_en_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_en_elementary_mathematics.yaml |   5 +
 .../en/global_mmlu_full_en_formal_logic.yaml  |   5 +
 .../en/global_mmlu_full_en_global_facts.yaml  |   5 +
 ...obal_mmlu_full_en_high_school_biology.yaml |   5 +
 ...al_mmlu_full_en_high_school_chemistry.yaml |   5 +
 ..._full_en_high_school_computer_science.yaml |   5 +
 ..._full_en_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_en_high_school_geography.yaml |   5 +
 ...n_high_school_government_and_politics.yaml |   5 +
 ...lu_full_en_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_en_high_school_mathematics.yaml |   5 +
 ...lu_full_en_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_en_high_school_physics.yaml |   5 +
 ...l_mmlu_full_en_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_en_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_en_high_school_us_history.yaml |   5 +
 ...mlu_full_en_high_school_world_history.yaml |   5 +
 .../en/global_mmlu_full_en_human_aging.yaml   |   5 +
 .../global_mmlu_full_en_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_en_international_law.yaml |   5 +
 .../en/global_mmlu_full_en_jurisprudence.yaml |   5 +
 ...global_mmlu_full_en_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_en_machine_learning.yaml |   5 +
 .../en/global_mmlu_full_en_management.yaml    |   5 +
 .../en/global_mmlu_full_en_marketing.yaml     |   5 +
 .../global_mmlu_full_en_medical_genetics.yaml |   5 +
 .../en/global_mmlu_full_en_miscellaneous.yaml |   5 +
 .../global_mmlu_full_en_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_en_moral_scenarios.yaml  |   5 +
 .../en/global_mmlu_full_en_nutrition.yaml     |   5 +
 .../en/global_mmlu_full_en_philosophy.yaml    |   5 +
 .../en/global_mmlu_full_en_prehistory.yaml    |   5 +
 ..._mmlu_full_en_professional_accounting.yaml |   5 +
 .../global_mmlu_full_en_professional_law.yaml |   5 +
 ...al_mmlu_full_en_professional_medicine.yaml |   5 +
 ..._mmlu_full_en_professional_psychology.yaml |   5 +
 .../global_mmlu_full_en_public_relations.yaml |   5 +
 .../global_mmlu_full_en_security_studies.yaml |   5 +
 .../en/global_mmlu_full_en_sociology.yaml     |   5 +
 ...global_mmlu_full_en_us_foreign_policy.yaml |   5 +
 .../full/en/global_mmlu_full_en_virology.yaml |   5 +
 .../global_mmlu_full_en_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/en/utils.py  |  73 +++++
 .../global_mmlu/full/es/_es_template_yaml     |  16 ++
 .../full/es/_global_mmlu_full_es.yaml         |  11 +
 .../es/_global_mmlu_full_es_humanities.yaml   |   8 +
 .../full/es/_global_mmlu_full_es_other.yaml   |   8 +
 .../_global_mmlu_full_es_social_sciences.yaml |   8 +
 .../full/es/_global_mmlu_full_es_stem.yaml    |   8 +
 .../global_mmlu_full_es_abstract_algebra.yaml |   5 +
 .../full/es/global_mmlu_full_es_anatomy.yaml  |   5 +
 .../es/global_mmlu_full_es_astronomy.yaml     |   5 +
 .../global_mmlu_full_es_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_es_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_es_college_biology.yaml  |   5 +
 ...global_mmlu_full_es_college_chemistry.yaml |   5 +
 ...mmlu_full_es_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_es_college_mathematics.yaml |   5 +
 .../global_mmlu_full_es_college_medicine.yaml |   5 +
 .../global_mmlu_full_es_college_physics.yaml  |   5 +
 ...global_mmlu_full_es_computer_security.yaml |   5 +
 ...lobal_mmlu_full_es_conceptual_physics.yaml |   5 +
 .../es/global_mmlu_full_es_econometrics.yaml  |   5 +
 ...l_mmlu_full_es_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_es_elementary_mathematics.yaml |   5 +
 .../es/global_mmlu_full_es_formal_logic.yaml  |   5 +
 .../es/global_mmlu_full_es_global_facts.yaml  |   5 +
 ...obal_mmlu_full_es_high_school_biology.yaml |   5 +
 ...al_mmlu_full_es_high_school_chemistry.yaml |   5 +
 ..._full_es_high_school_computer_science.yaml |   5 +
 ..._full_es_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_es_high_school_geography.yaml |   5 +
 ...s_high_school_government_and_politics.yaml |   5 +
 ...lu_full_es_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_es_high_school_mathematics.yaml |   5 +
 ...lu_full_es_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_es_high_school_physics.yaml |   5 +
 ...l_mmlu_full_es_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_es_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_es_high_school_us_history.yaml |   5 +
 ...mlu_full_es_high_school_world_history.yaml |   5 +
 .../es/global_mmlu_full_es_human_aging.yaml   |   5 +
 .../global_mmlu_full_es_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_es_international_law.yaml |   5 +
 .../es/global_mmlu_full_es_jurisprudence.yaml |   5 +
 ...global_mmlu_full_es_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_es_machine_learning.yaml |   5 +
 .../es/global_mmlu_full_es_management.yaml    |   5 +
 .../es/global_mmlu_full_es_marketing.yaml     |   5 +
 .../global_mmlu_full_es_medical_genetics.yaml |   5 +
 .../es/global_mmlu_full_es_miscellaneous.yaml |   5 +
 .../global_mmlu_full_es_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_es_moral_scenarios.yaml  |   5 +
 .../es/global_mmlu_full_es_nutrition.yaml     |   5 +
 .../es/global_mmlu_full_es_philosophy.yaml    |   5 +
 .../es/global_mmlu_full_es_prehistory.yaml    |   5 +
 ..._mmlu_full_es_professional_accounting.yaml |   5 +
 .../global_mmlu_full_es_professional_law.yaml |   5 +
 ...al_mmlu_full_es_professional_medicine.yaml |   5 +
 ..._mmlu_full_es_professional_psychology.yaml |   5 +
 .../global_mmlu_full_es_public_relations.yaml |   5 +
 .../global_mmlu_full_es_security_studies.yaml |   5 +
 .../es/global_mmlu_full_es_sociology.yaml     |   5 +
 ...global_mmlu_full_es_us_foreign_policy.yaml |   5 +
 .../full/es/global_mmlu_full_es_virology.yaml |   5 +
 .../global_mmlu_full_es_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/es/utils.py  |  73 +++++
 .../global_mmlu/full/fa/_fa_template_yaml     |  16 ++
 .../full/fa/_global_mmlu_full_fa.yaml         |  11 +
 .../fa/_global_mmlu_full_fa_humanities.yaml   |   8 +
 .../full/fa/_global_mmlu_full_fa_other.yaml   |   8 +
 .../_global_mmlu_full_fa_social_sciences.yaml |   8 +
 .../full/fa/_global_mmlu_full_fa_stem.yaml    |   8 +
 .../global_mmlu_full_fa_abstract_algebra.yaml |   5 +
 .../full/fa/global_mmlu_full_fa_anatomy.yaml  |   5 +
 .../fa/global_mmlu_full_fa_astronomy.yaml     |   5 +
 .../global_mmlu_full_fa_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_fa_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_fa_college_biology.yaml  |   5 +
 ...global_mmlu_full_fa_college_chemistry.yaml |   5 +
 ...mmlu_full_fa_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_fa_college_mathematics.yaml |   5 +
 .../global_mmlu_full_fa_college_medicine.yaml |   5 +
 .../global_mmlu_full_fa_college_physics.yaml  |   5 +
 ...global_mmlu_full_fa_computer_security.yaml |   5 +
 ...lobal_mmlu_full_fa_conceptual_physics.yaml |   5 +
 .../fa/global_mmlu_full_fa_econometrics.yaml  |   5 +
 ...l_mmlu_full_fa_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_fa_elementary_mathematics.yaml |   5 +
 .../fa/global_mmlu_full_fa_formal_logic.yaml  |   5 +
 .../fa/global_mmlu_full_fa_global_facts.yaml  |   5 +
 ...obal_mmlu_full_fa_high_school_biology.yaml |   5 +
 ...al_mmlu_full_fa_high_school_chemistry.yaml |   5 +
 ..._full_fa_high_school_computer_science.yaml |   5 +
 ..._full_fa_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_fa_high_school_geography.yaml |   5 +
 ...a_high_school_government_and_politics.yaml |   5 +
 ...lu_full_fa_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_fa_high_school_mathematics.yaml |   5 +
 ...lu_full_fa_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_fa_high_school_physics.yaml |   5 +
 ...l_mmlu_full_fa_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_fa_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_fa_high_school_us_history.yaml |   5 +
 ...mlu_full_fa_high_school_world_history.yaml |   5 +
 .../fa/global_mmlu_full_fa_human_aging.yaml   |   5 +
 .../global_mmlu_full_fa_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_fa_international_law.yaml |   5 +
 .../fa/global_mmlu_full_fa_jurisprudence.yaml |   5 +
 ...global_mmlu_full_fa_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_fa_machine_learning.yaml |   5 +
 .../fa/global_mmlu_full_fa_management.yaml    |   5 +
 .../fa/global_mmlu_full_fa_marketing.yaml     |   5 +
 .../global_mmlu_full_fa_medical_genetics.yaml |   5 +
 .../fa/global_mmlu_full_fa_miscellaneous.yaml |   5 +
 .../global_mmlu_full_fa_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_fa_moral_scenarios.yaml  |   5 +
 .../fa/global_mmlu_full_fa_nutrition.yaml     |   5 +
 .../fa/global_mmlu_full_fa_philosophy.yaml    |   5 +
 .../fa/global_mmlu_full_fa_prehistory.yaml    |   5 +
 ..._mmlu_full_fa_professional_accounting.yaml |   5 +
 .../global_mmlu_full_fa_professional_law.yaml |   5 +
 ...al_mmlu_full_fa_professional_medicine.yaml |   5 +
 ..._mmlu_full_fa_professional_psychology.yaml |   5 +
 .../global_mmlu_full_fa_public_relations.yaml |   5 +
 .../global_mmlu_full_fa_security_studies.yaml |   5 +
 .../fa/global_mmlu_full_fa_sociology.yaml     |   5 +
 ...global_mmlu_full_fa_us_foreign_policy.yaml |   5 +
 .../full/fa/global_mmlu_full_fa_virology.yaml |   5 +
 .../global_mmlu_full_fa_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/fa/utils.py  |  73 +++++
 .../global_mmlu/full/fil/_fil_template_yaml   |  16 ++
 .../full/fil/_global_mmlu_full_fil.yaml       |  11 +
 .../fil/_global_mmlu_full_fil_humanities.yaml |   8 +
 .../full/fil/_global_mmlu_full_fil_other.yaml |   8 +
 ..._global_mmlu_full_fil_social_sciences.yaml |   8 +
 .../full/fil/_global_mmlu_full_fil_stem.yaml  |   8 +
 ...global_mmlu_full_fil_abstract_algebra.yaml |   5 +
 .../fil/global_mmlu_full_fil_anatomy.yaml     |   5 +
 .../fil/global_mmlu_full_fil_astronomy.yaml   |   5 +
 .../global_mmlu_full_fil_business_ethics.yaml |   5 +
 ...obal_mmlu_full_fil_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_fil_college_biology.yaml |   5 +
 ...lobal_mmlu_full_fil_college_chemistry.yaml |   5 +
 ...mlu_full_fil_college_computer_science.yaml |   5 +
 ...bal_mmlu_full_fil_college_mathematics.yaml |   5 +
 ...global_mmlu_full_fil_college_medicine.yaml |   5 +
 .../global_mmlu_full_fil_college_physics.yaml |   5 +
 ...lobal_mmlu_full_fil_computer_security.yaml |   5 +
 ...obal_mmlu_full_fil_conceptual_physics.yaml |   5 +
 .../global_mmlu_full_fil_econometrics.yaml    |   5 +
 ..._mmlu_full_fil_electrical_engineering.yaml |   5 +
 ..._mmlu_full_fil_elementary_mathematics.yaml |   5 +
 .../global_mmlu_full_fil_formal_logic.yaml    |   5 +
 .../global_mmlu_full_fil_global_facts.yaml    |   5 +
 ...bal_mmlu_full_fil_high_school_biology.yaml |   5 +
 ...l_mmlu_full_fil_high_school_chemistry.yaml |   5 +
 ...full_fil_high_school_computer_science.yaml |   5 +
 ...full_fil_high_school_european_history.yaml |   5 +
 ...l_mmlu_full_fil_high_school_geography.yaml |   5 +
 ...l_high_school_government_and_politics.yaml |   5 +
 ...u_full_fil_high_school_macroeconomics.yaml |   5 +
 ...mmlu_full_fil_high_school_mathematics.yaml |   5 +
 ...u_full_fil_high_school_microeconomics.yaml |   5 +
 ...bal_mmlu_full_fil_high_school_physics.yaml |   5 +
 ..._mmlu_full_fil_high_school_psychology.yaml |   5 +
 ..._mmlu_full_fil_high_school_statistics.yaml |   5 +
 ..._mmlu_full_fil_high_school_us_history.yaml |   5 +
 ...lu_full_fil_high_school_world_history.yaml |   5 +
 .../fil/global_mmlu_full_fil_human_aging.yaml |   5 +
 .../global_mmlu_full_fil_human_sexuality.yaml |   5 +
 ...lobal_mmlu_full_fil_international_law.yaml |   5 +
 .../global_mmlu_full_fil_jurisprudence.yaml   |   5 +
 ...lobal_mmlu_full_fil_logical_fallacies.yaml |   5 +
 ...global_mmlu_full_fil_machine_learning.yaml |   5 +
 .../fil/global_mmlu_full_fil_management.yaml  |   5 +
 .../fil/global_mmlu_full_fil_marketing.yaml   |   5 +
 ...global_mmlu_full_fil_medical_genetics.yaml |   5 +
 .../global_mmlu_full_fil_miscellaneous.yaml   |   5 +
 .../global_mmlu_full_fil_moral_disputes.yaml  |   5 +
 .../global_mmlu_full_fil_moral_scenarios.yaml |   5 +
 .../fil/global_mmlu_full_fil_nutrition.yaml   |   5 +
 .../fil/global_mmlu_full_fil_philosophy.yaml  |   5 +
 .../fil/global_mmlu_full_fil_prehistory.yaml  |   5 +
 ...mmlu_full_fil_professional_accounting.yaml |   5 +
 ...global_mmlu_full_fil_professional_law.yaml |   5 +
 ...l_mmlu_full_fil_professional_medicine.yaml |   5 +
 ...mmlu_full_fil_professional_psychology.yaml |   5 +
 ...global_mmlu_full_fil_public_relations.yaml |   5 +
 ...global_mmlu_full_fil_security_studies.yaml |   5 +
 .../fil/global_mmlu_full_fil_sociology.yaml   |   5 +
 ...lobal_mmlu_full_fil_us_foreign_policy.yaml |   5 +
 .../fil/global_mmlu_full_fil_virology.yaml    |   5 +
 .../global_mmlu_full_fil_world_religions.yaml |   5 +
 lmms_eval/tasks/global_mmlu/full/fil/utils.py |  73 +++++
 .../global_mmlu/full/fr/_fr_template_yaml     |  16 ++
 .../full/fr/_global_mmlu_full_fr.yaml         |  11 +
 .../fr/_global_mmlu_full_fr_humanities.yaml   |   8 +
 .../full/fr/_global_mmlu_full_fr_other.yaml   |   8 +
 .../_global_mmlu_full_fr_social_sciences.yaml |   8 +
 .../full/fr/_global_mmlu_full_fr_stem.yaml    |   8 +
 .../global_mmlu_full_fr_abstract_algebra.yaml |   5 +
 .../full/fr/global_mmlu_full_fr_anatomy.yaml  |   5 +
 .../fr/global_mmlu_full_fr_astronomy.yaml     |   5 +
 .../global_mmlu_full_fr_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_fr_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_fr_college_biology.yaml  |   5 +
 ...global_mmlu_full_fr_college_chemistry.yaml |   5 +
 ...mmlu_full_fr_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_fr_college_mathematics.yaml |   5 +
 .../global_mmlu_full_fr_college_medicine.yaml |   5 +
 .../global_mmlu_full_fr_college_physics.yaml  |   5 +
 ...global_mmlu_full_fr_computer_security.yaml |   5 +
 ...lobal_mmlu_full_fr_conceptual_physics.yaml |   5 +
 .../fr/global_mmlu_full_fr_econometrics.yaml  |   5 +
 ...l_mmlu_full_fr_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_fr_elementary_mathematics.yaml |   5 +
 .../fr/global_mmlu_full_fr_formal_logic.yaml  |   5 +
 .../fr/global_mmlu_full_fr_global_facts.yaml  |   5 +
 ...obal_mmlu_full_fr_high_school_biology.yaml |   5 +
 ...al_mmlu_full_fr_high_school_chemistry.yaml |   5 +
 ..._full_fr_high_school_computer_science.yaml |   5 +
 ..._full_fr_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_fr_high_school_geography.yaml |   5 +
 ...r_high_school_government_and_politics.yaml |   5 +
 ...lu_full_fr_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_fr_high_school_mathematics.yaml |   5 +
 ...lu_full_fr_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_fr_high_school_physics.yaml |   5 +
 ...l_mmlu_full_fr_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_fr_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_fr_high_school_us_history.yaml |   5 +
 ...mlu_full_fr_high_school_world_history.yaml |   5 +
 .../fr/global_mmlu_full_fr_human_aging.yaml   |   5 +
 .../global_mmlu_full_fr_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_fr_international_law.yaml |   5 +
 .../fr/global_mmlu_full_fr_jurisprudence.yaml |   5 +
 ...global_mmlu_full_fr_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_fr_machine_learning.yaml |   5 +
 .../fr/global_mmlu_full_fr_management.yaml    |   5 +
 .../fr/global_mmlu_full_fr_marketing.yaml     |   5 +
 .../global_mmlu_full_fr_medical_genetics.yaml |   5 +
 .../fr/global_mmlu_full_fr_miscellaneous.yaml |   5 +
 .../global_mmlu_full_fr_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_fr_moral_scenarios.yaml  |   5 +
 .../fr/global_mmlu_full_fr_nutrition.yaml     |   5 +
 .../fr/global_mmlu_full_fr_philosophy.yaml    |   5 +
 .../fr/global_mmlu_full_fr_prehistory.yaml    |   5 +
 ..._mmlu_full_fr_professional_accounting.yaml |   5 +
 .../global_mmlu_full_fr_professional_law.yaml |   5 +
 ...al_mmlu_full_fr_professional_medicine.yaml |   5 +
 ..._mmlu_full_fr_professional_psychology.yaml |   5 +
 .../global_mmlu_full_fr_public_relations.yaml |   5 +
 .../global_mmlu_full_fr_security_studies.yaml |   5 +
 .../fr/global_mmlu_full_fr_sociology.yaml     |   5 +
 ...global_mmlu_full_fr_us_foreign_policy.yaml |   5 +
 .../full/fr/global_mmlu_full_fr_virology.yaml |   5 +
 .../global_mmlu_full_fr_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/fr/utils.py  |  73 +++++
 .../full/ha/_global_mmlu_full_ha.yaml         |  11 +
 .../ha/_global_mmlu_full_ha_humanities.yaml   |   8 +
 .../full/ha/_global_mmlu_full_ha_other.yaml   |   8 +
 .../_global_mmlu_full_ha_social_sciences.yaml |   8 +
 .../full/ha/_global_mmlu_full_ha_stem.yaml    |   8 +
 .../global_mmlu/full/ha/_ha_template_yaml     |  16 ++
 .../global_mmlu_full_ha_abstract_algebra.yaml |   5 +
 .../full/ha/global_mmlu_full_ha_anatomy.yaml  |   5 +
 .../ha/global_mmlu_full_ha_astronomy.yaml     |   5 +
 .../global_mmlu_full_ha_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_ha_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_ha_college_biology.yaml  |   5 +
 ...global_mmlu_full_ha_college_chemistry.yaml |   5 +
 ...mmlu_full_ha_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_ha_college_mathematics.yaml |   5 +
 .../global_mmlu_full_ha_college_medicine.yaml |   5 +
 .../global_mmlu_full_ha_college_physics.yaml  |   5 +
 ...global_mmlu_full_ha_computer_security.yaml |   5 +
 ...lobal_mmlu_full_ha_conceptual_physics.yaml |   5 +
 .../ha/global_mmlu_full_ha_econometrics.yaml  |   5 +
 ...l_mmlu_full_ha_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_ha_elementary_mathematics.yaml |   5 +
 .../ha/global_mmlu_full_ha_formal_logic.yaml  |   5 +
 .../ha/global_mmlu_full_ha_global_facts.yaml  |   5 +
 ...obal_mmlu_full_ha_high_school_biology.yaml |   5 +
 ...al_mmlu_full_ha_high_school_chemistry.yaml |   5 +
 ..._full_ha_high_school_computer_science.yaml |   5 +
 ..._full_ha_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_ha_high_school_geography.yaml |   5 +
 ...a_high_school_government_and_politics.yaml |   5 +
 ...lu_full_ha_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_ha_high_school_mathematics.yaml |   5 +
 ...lu_full_ha_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_ha_high_school_physics.yaml |   5 +
 ...l_mmlu_full_ha_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_ha_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_ha_high_school_us_history.yaml |   5 +
 ...mlu_full_ha_high_school_world_history.yaml |   5 +
 .../ha/global_mmlu_full_ha_human_aging.yaml   |   5 +
 .../global_mmlu_full_ha_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_ha_international_law.yaml |   5 +
 .../ha/global_mmlu_full_ha_jurisprudence.yaml |   5 +
 ...global_mmlu_full_ha_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_ha_machine_learning.yaml |   5 +
 .../ha/global_mmlu_full_ha_management.yaml    |   5 +
 .../ha/global_mmlu_full_ha_marketing.yaml     |   5 +
 .../global_mmlu_full_ha_medical_genetics.yaml |   5 +
 .../ha/global_mmlu_full_ha_miscellaneous.yaml |   5 +
 .../global_mmlu_full_ha_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_ha_moral_scenarios.yaml  |   5 +
 .../ha/global_mmlu_full_ha_nutrition.yaml     |   5 +
 .../ha/global_mmlu_full_ha_philosophy.yaml    |   5 +
 .../ha/global_mmlu_full_ha_prehistory.yaml    |   5 +
 ..._mmlu_full_ha_professional_accounting.yaml |   5 +
 .../global_mmlu_full_ha_professional_law.yaml |   5 +
 ...al_mmlu_full_ha_professional_medicine.yaml |   5 +
 ..._mmlu_full_ha_professional_psychology.yaml |   5 +
 .../global_mmlu_full_ha_public_relations.yaml |   5 +
 .../global_mmlu_full_ha_security_studies.yaml |   5 +
 .../ha/global_mmlu_full_ha_sociology.yaml     |   5 +
 ...global_mmlu_full_ha_us_foreign_policy.yaml |   5 +
 .../full/ha/global_mmlu_full_ha_virology.yaml |   5 +
 .../global_mmlu_full_ha_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/ha/utils.py  |  73 +++++
 .../full/he/_global_mmlu_full_he.yaml         |  11 +
 .../he/_global_mmlu_full_he_humanities.yaml   |   8 +
 .../full/he/_global_mmlu_full_he_other.yaml   |   8 +
 .../_global_mmlu_full_he_social_sciences.yaml |   8 +
 .../full/he/_global_mmlu_full_he_stem.yaml    |   8 +
 .../global_mmlu/full/he/_he_template_yaml     |  16 ++
 .../global_mmlu_full_he_abstract_algebra.yaml |   5 +
 .../full/he/global_mmlu_full_he_anatomy.yaml  |   5 +
 .../he/global_mmlu_full_he_astronomy.yaml     |   5 +
 .../global_mmlu_full_he_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_he_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_he_college_biology.yaml  |   5 +
 ...global_mmlu_full_he_college_chemistry.yaml |   5 +
 ...mmlu_full_he_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_he_college_mathematics.yaml |   5 +
 .../global_mmlu_full_he_college_medicine.yaml |   5 +
 .../global_mmlu_full_he_college_physics.yaml  |   5 +
 ...global_mmlu_full_he_computer_security.yaml |   5 +
 ...lobal_mmlu_full_he_conceptual_physics.yaml |   5 +
 .../he/global_mmlu_full_he_econometrics.yaml  |   5 +
 ...l_mmlu_full_he_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_he_elementary_mathematics.yaml |   5 +
 .../he/global_mmlu_full_he_formal_logic.yaml  |   5 +
 .../he/global_mmlu_full_he_global_facts.yaml  |   5 +
 ...obal_mmlu_full_he_high_school_biology.yaml |   5 +
 ...al_mmlu_full_he_high_school_chemistry.yaml |   5 +
 ..._full_he_high_school_computer_science.yaml |   5 +
 ..._full_he_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_he_high_school_geography.yaml |   5 +
 ...e_high_school_government_and_politics.yaml |   5 +
 ...lu_full_he_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_he_high_school_mathematics.yaml |   5 +
 ...lu_full_he_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_he_high_school_physics.yaml |   5 +
 ...l_mmlu_full_he_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_he_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_he_high_school_us_history.yaml |   5 +
 ...mlu_full_he_high_school_world_history.yaml |   5 +
 .../he/global_mmlu_full_he_human_aging.yaml   |   5 +
 .../global_mmlu_full_he_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_he_international_law.yaml |   5 +
 .../he/global_mmlu_full_he_jurisprudence.yaml |   5 +
 ...global_mmlu_full_he_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_he_machine_learning.yaml |   5 +
 .../he/global_mmlu_full_he_management.yaml    |   5 +
 .../he/global_mmlu_full_he_marketing.yaml     |   5 +
 .../global_mmlu_full_he_medical_genetics.yaml |   5 +
 .../he/global_mmlu_full_he_miscellaneous.yaml |   5 +
 .../global_mmlu_full_he_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_he_moral_scenarios.yaml  |   5 +
 .../he/global_mmlu_full_he_nutrition.yaml     |   5 +
 .../he/global_mmlu_full_he_philosophy.yaml    |   5 +
 .../he/global_mmlu_full_he_prehistory.yaml    |   5 +
 ..._mmlu_full_he_professional_accounting.yaml |   5 +
 .../global_mmlu_full_he_professional_law.yaml |   5 +
 ...al_mmlu_full_he_professional_medicine.yaml |   5 +
 ..._mmlu_full_he_professional_psychology.yaml |   5 +
 .../global_mmlu_full_he_public_relations.yaml |   5 +
 .../global_mmlu_full_he_security_studies.yaml |   5 +
 .../he/global_mmlu_full_he_sociology.yaml     |   5 +
 ...global_mmlu_full_he_us_foreign_policy.yaml |   5 +
 .../full/he/global_mmlu_full_he_virology.yaml |   5 +
 .../global_mmlu_full_he_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/he/utils.py  |  73 +++++
 .../full/hi/_global_mmlu_full_hi.yaml         |  11 +
 .../hi/_global_mmlu_full_hi_humanities.yaml   |   8 +
 .../full/hi/_global_mmlu_full_hi_other.yaml   |   8 +
 .../_global_mmlu_full_hi_social_sciences.yaml |   8 +
 .../full/hi/_global_mmlu_full_hi_stem.yaml    |   8 +
 .../global_mmlu/full/hi/_hi_template_yaml     |  16 ++
 .../global_mmlu_full_hi_abstract_algebra.yaml |   5 +
 .../full/hi/global_mmlu_full_hi_anatomy.yaml  |   5 +
 .../hi/global_mmlu_full_hi_astronomy.yaml     |   5 +
 .../global_mmlu_full_hi_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_hi_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_hi_college_biology.yaml  |   5 +
 ...global_mmlu_full_hi_college_chemistry.yaml |   5 +
 ...mmlu_full_hi_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_hi_college_mathematics.yaml |   5 +
 .../global_mmlu_full_hi_college_medicine.yaml |   5 +
 .../global_mmlu_full_hi_college_physics.yaml  |   5 +
 ...global_mmlu_full_hi_computer_security.yaml |   5 +
 ...lobal_mmlu_full_hi_conceptual_physics.yaml |   5 +
 .../hi/global_mmlu_full_hi_econometrics.yaml  |   5 +
 ...l_mmlu_full_hi_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_hi_elementary_mathematics.yaml |   5 +
 .../hi/global_mmlu_full_hi_formal_logic.yaml  |   5 +
 .../hi/global_mmlu_full_hi_global_facts.yaml  |   5 +
 ...obal_mmlu_full_hi_high_school_biology.yaml |   5 +
 ...al_mmlu_full_hi_high_school_chemistry.yaml |   5 +
 ..._full_hi_high_school_computer_science.yaml |   5 +
 ..._full_hi_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_hi_high_school_geography.yaml |   5 +
 ...i_high_school_government_and_politics.yaml |   5 +
 ...lu_full_hi_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_hi_high_school_mathematics.yaml |   5 +
 ...lu_full_hi_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_hi_high_school_physics.yaml |   5 +
 ...l_mmlu_full_hi_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_hi_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_hi_high_school_us_history.yaml |   5 +
 ...mlu_full_hi_high_school_world_history.yaml |   5 +
 .../hi/global_mmlu_full_hi_human_aging.yaml   |   5 +
 .../global_mmlu_full_hi_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_hi_international_law.yaml |   5 +
 .../hi/global_mmlu_full_hi_jurisprudence.yaml |   5 +
 ...global_mmlu_full_hi_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_hi_machine_learning.yaml |   5 +
 .../hi/global_mmlu_full_hi_management.yaml    |   5 +
 .../hi/global_mmlu_full_hi_marketing.yaml     |   5 +
 .../global_mmlu_full_hi_medical_genetics.yaml |   5 +
 .../hi/global_mmlu_full_hi_miscellaneous.yaml |   5 +
 .../global_mmlu_full_hi_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_hi_moral_scenarios.yaml  |   5 +
 .../hi/global_mmlu_full_hi_nutrition.yaml     |   5 +
 .../hi/global_mmlu_full_hi_philosophy.yaml    |   5 +
 .../hi/global_mmlu_full_hi_prehistory.yaml    |   5 +
 ..._mmlu_full_hi_professional_accounting.yaml |   5 +
 .../global_mmlu_full_hi_professional_law.yaml |   5 +
 ...al_mmlu_full_hi_professional_medicine.yaml |   5 +
 ..._mmlu_full_hi_professional_psychology.yaml |   5 +
 .../global_mmlu_full_hi_public_relations.yaml |   5 +
 .../global_mmlu_full_hi_security_studies.yaml |   5 +
 .../hi/global_mmlu_full_hi_sociology.yaml     |   5 +
 ...global_mmlu_full_hi_us_foreign_policy.yaml |   5 +
 .../full/hi/global_mmlu_full_hi_virology.yaml |   5 +
 .../global_mmlu_full_hi_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/hi/utils.py  |  73 +++++
 .../full/id/_global_mmlu_full_id.yaml         |  11 +
 .../id/_global_mmlu_full_id_humanities.yaml   |   8 +
 .../full/id/_global_mmlu_full_id_other.yaml   |   8 +
 .../_global_mmlu_full_id_social_sciences.yaml |   8 +
 .../full/id/_global_mmlu_full_id_stem.yaml    |   8 +
 .../global_mmlu/full/id/_id_template_yaml     |  16 ++
 .../global_mmlu_full_id_abstract_algebra.yaml |   5 +
 .../full/id/global_mmlu_full_id_anatomy.yaml  |   5 +
 .../id/global_mmlu_full_id_astronomy.yaml     |   5 +
 .../global_mmlu_full_id_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_id_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_id_college_biology.yaml  |   5 +
 ...global_mmlu_full_id_college_chemistry.yaml |   5 +
 ...mmlu_full_id_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_id_college_mathematics.yaml |   5 +
 .../global_mmlu_full_id_college_medicine.yaml |   5 +
 .../global_mmlu_full_id_college_physics.yaml  |   5 +
 ...global_mmlu_full_id_computer_security.yaml |   5 +
 ...lobal_mmlu_full_id_conceptual_physics.yaml |   5 +
 .../id/global_mmlu_full_id_econometrics.yaml  |   5 +
 ...l_mmlu_full_id_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_id_elementary_mathematics.yaml |   5 +
 .../id/global_mmlu_full_id_formal_logic.yaml  |   5 +
 .../id/global_mmlu_full_id_global_facts.yaml  |   5 +
 ...obal_mmlu_full_id_high_school_biology.yaml |   5 +
 ...al_mmlu_full_id_high_school_chemistry.yaml |   5 +
 ..._full_id_high_school_computer_science.yaml |   5 +
 ..._full_id_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_id_high_school_geography.yaml |   5 +
 ...d_high_school_government_and_politics.yaml |   5 +
 ...lu_full_id_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_id_high_school_mathematics.yaml |   5 +
 ...lu_full_id_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_id_high_school_physics.yaml |   5 +
 ...l_mmlu_full_id_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_id_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_id_high_school_us_history.yaml |   5 +
 ...mlu_full_id_high_school_world_history.yaml |   5 +
 .../id/global_mmlu_full_id_human_aging.yaml   |   5 +
 .../global_mmlu_full_id_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_id_international_law.yaml |   5 +
 .../id/global_mmlu_full_id_jurisprudence.yaml |   5 +
 ...global_mmlu_full_id_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_id_machine_learning.yaml |   5 +
 .../id/global_mmlu_full_id_management.yaml    |   5 +
 .../id/global_mmlu_full_id_marketing.yaml     |   5 +
 .../global_mmlu_full_id_medical_genetics.yaml |   5 +
 .../id/global_mmlu_full_id_miscellaneous.yaml |   5 +
 .../global_mmlu_full_id_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_id_moral_scenarios.yaml  |   5 +
 .../id/global_mmlu_full_id_nutrition.yaml     |   5 +
 .../id/global_mmlu_full_id_philosophy.yaml    |   5 +
 .../id/global_mmlu_full_id_prehistory.yaml    |   5 +
 ..._mmlu_full_id_professional_accounting.yaml |   5 +
 .../global_mmlu_full_id_professional_law.yaml |   5 +
 ...al_mmlu_full_id_professional_medicine.yaml |   5 +
 ..._mmlu_full_id_professional_psychology.yaml |   5 +
 .../global_mmlu_full_id_public_relations.yaml |   5 +
 .../global_mmlu_full_id_security_studies.yaml |   5 +
 .../id/global_mmlu_full_id_sociology.yaml     |   5 +
 ...global_mmlu_full_id_us_foreign_policy.yaml |   5 +
 .../full/id/global_mmlu_full_id_virology.yaml |   5 +
 .../global_mmlu_full_id_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/id/utils.py  |  73 +++++
 .../full/ig/_global_mmlu_full_ig.yaml         |  11 +
 .../ig/_global_mmlu_full_ig_humanities.yaml   |   8 +
 .../full/ig/_global_mmlu_full_ig_other.yaml   |   8 +
 .../_global_mmlu_full_ig_social_sciences.yaml |   8 +
 .../full/ig/_global_mmlu_full_ig_stem.yaml    |   8 +
 .../global_mmlu/full/ig/_ig_template_yaml     |  16 ++
 .../global_mmlu_full_ig_abstract_algebra.yaml |   5 +
 .../full/ig/global_mmlu_full_ig_anatomy.yaml  |   5 +
 .../ig/global_mmlu_full_ig_astronomy.yaml     |   5 +
 .../global_mmlu_full_ig_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_ig_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_ig_college_biology.yaml  |   5 +
 ...global_mmlu_full_ig_college_chemistry.yaml |   5 +
 ...mmlu_full_ig_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_ig_college_mathematics.yaml |   5 +
 .../global_mmlu_full_ig_college_medicine.yaml |   5 +
 .../global_mmlu_full_ig_college_physics.yaml  |   5 +
 ...global_mmlu_full_ig_computer_security.yaml |   5 +
 ...lobal_mmlu_full_ig_conceptual_physics.yaml |   5 +
 .../ig/global_mmlu_full_ig_econometrics.yaml  |   5 +
 ...l_mmlu_full_ig_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_ig_elementary_mathematics.yaml |   5 +
 .../ig/global_mmlu_full_ig_formal_logic.yaml  |   5 +
 .../ig/global_mmlu_full_ig_global_facts.yaml  |   5 +
 ...obal_mmlu_full_ig_high_school_biology.yaml |   5 +
 ...al_mmlu_full_ig_high_school_chemistry.yaml |   5 +
 ..._full_ig_high_school_computer_science.yaml |   5 +
 ..._full_ig_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_ig_high_school_geography.yaml |   5 +
 ...g_high_school_government_and_politics.yaml |   5 +
 ...lu_full_ig_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_ig_high_school_mathematics.yaml |   5 +
 ...lu_full_ig_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_ig_high_school_physics.yaml |   5 +
 ...l_mmlu_full_ig_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_ig_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_ig_high_school_us_history.yaml |   5 +
 ...mlu_full_ig_high_school_world_history.yaml |   5 +
 .../ig/global_mmlu_full_ig_human_aging.yaml   |   5 +
 .../global_mmlu_full_ig_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_ig_international_law.yaml |   5 +
 .../ig/global_mmlu_full_ig_jurisprudence.yaml |   5 +
 ...global_mmlu_full_ig_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_ig_machine_learning.yaml |   5 +
 .../ig/global_mmlu_full_ig_management.yaml    |   5 +
 .../ig/global_mmlu_full_ig_marketing.yaml     |   5 +
 .../global_mmlu_full_ig_medical_genetics.yaml |   5 +
 .../ig/global_mmlu_full_ig_miscellaneous.yaml |   5 +
 .../global_mmlu_full_ig_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_ig_moral_scenarios.yaml  |   5 +
 .../ig/global_mmlu_full_ig_nutrition.yaml     |   5 +
 .../ig/global_mmlu_full_ig_philosophy.yaml    |   5 +
 .../ig/global_mmlu_full_ig_prehistory.yaml    |   5 +
 ..._mmlu_full_ig_professional_accounting.yaml |   5 +
 .../global_mmlu_full_ig_professional_law.yaml |   5 +
 ...al_mmlu_full_ig_professional_medicine.yaml |   5 +
 ..._mmlu_full_ig_professional_psychology.yaml |   5 +
 .../global_mmlu_full_ig_public_relations.yaml |   5 +
 .../global_mmlu_full_ig_security_studies.yaml |   5 +
 .../ig/global_mmlu_full_ig_sociology.yaml     |   5 +
 ...global_mmlu_full_ig_us_foreign_policy.yaml |   5 +
 .../full/ig/global_mmlu_full_ig_virology.yaml |   5 +
 .../global_mmlu_full_ig_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/ig/utils.py  |  73 +++++
 .../full/it/_global_mmlu_full_it.yaml         |  11 +
 .../it/_global_mmlu_full_it_humanities.yaml   |   8 +
 .../full/it/_global_mmlu_full_it_other.yaml   |   8 +
 .../_global_mmlu_full_it_social_sciences.yaml |   8 +
 .../full/it/_global_mmlu_full_it_stem.yaml    |   8 +
 .../global_mmlu/full/it/_it_template_yaml     |  16 ++
 .../global_mmlu_full_it_abstract_algebra.yaml |   5 +
 .../full/it/global_mmlu_full_it_anatomy.yaml  |   5 +
 .../it/global_mmlu_full_it_astronomy.yaml     |   5 +
 .../global_mmlu_full_it_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_it_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_it_college_biology.yaml  |   5 +
 ...global_mmlu_full_it_college_chemistry.yaml |   5 +
 ...mmlu_full_it_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_it_college_mathematics.yaml |   5 +
 .../global_mmlu_full_it_college_medicine.yaml |   5 +
 .../global_mmlu_full_it_college_physics.yaml  |   5 +
 ...global_mmlu_full_it_computer_security.yaml |   5 +
 ...lobal_mmlu_full_it_conceptual_physics.yaml |   5 +
 .../it/global_mmlu_full_it_econometrics.yaml  |   5 +
 ...l_mmlu_full_it_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_it_elementary_mathematics.yaml |   5 +
 .../it/global_mmlu_full_it_formal_logic.yaml  |   5 +
 .../it/global_mmlu_full_it_global_facts.yaml  |   5 +
 ...obal_mmlu_full_it_high_school_biology.yaml |   5 +
 ...al_mmlu_full_it_high_school_chemistry.yaml |   5 +
 ..._full_it_high_school_computer_science.yaml |   5 +
 ..._full_it_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_it_high_school_geography.yaml |   5 +
 ...t_high_school_government_and_politics.yaml |   5 +
 ...lu_full_it_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_it_high_school_mathematics.yaml |   5 +
 ...lu_full_it_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_it_high_school_physics.yaml |   5 +
 ...l_mmlu_full_it_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_it_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_it_high_school_us_history.yaml |   5 +
 ...mlu_full_it_high_school_world_history.yaml |   5 +
 .../it/global_mmlu_full_it_human_aging.yaml   |   5 +
 .../global_mmlu_full_it_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_it_international_law.yaml |   5 +
 .../it/global_mmlu_full_it_jurisprudence.yaml |   5 +
 ...global_mmlu_full_it_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_it_machine_learning.yaml |   5 +
 .../it/global_mmlu_full_it_management.yaml    |   5 +
 .../it/global_mmlu_full_it_marketing.yaml     |   5 +
 .../global_mmlu_full_it_medical_genetics.yaml |   5 +
 .../it/global_mmlu_full_it_miscellaneous.yaml |   5 +
 .../global_mmlu_full_it_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_it_moral_scenarios.yaml  |   5 +
 .../it/global_mmlu_full_it_nutrition.yaml     |   5 +
 .../it/global_mmlu_full_it_philosophy.yaml    |   5 +
 .../it/global_mmlu_full_it_prehistory.yaml    |   5 +
 ..._mmlu_full_it_professional_accounting.yaml |   5 +
 .../global_mmlu_full_it_professional_law.yaml |   5 +
 ...al_mmlu_full_it_professional_medicine.yaml |   5 +
 ..._mmlu_full_it_professional_psychology.yaml |   5 +
 .../global_mmlu_full_it_public_relations.yaml |   5 +
 .../global_mmlu_full_it_security_studies.yaml |   5 +
 .../it/global_mmlu_full_it_sociology.yaml     |   5 +
 ...global_mmlu_full_it_us_foreign_policy.yaml |   5 +
 .../full/it/global_mmlu_full_it_virology.yaml |   5 +
 .../global_mmlu_full_it_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/it/utils.py  |  73 +++++
 .../full/ja/_global_mmlu_full_ja.yaml         |  11 +
 .../ja/_global_mmlu_full_ja_humanities.yaml   |   8 +
 .../full/ja/_global_mmlu_full_ja_other.yaml   |   8 +
 .../_global_mmlu_full_ja_social_sciences.yaml |   8 +
 .../full/ja/_global_mmlu_full_ja_stem.yaml    |   8 +
 .../global_mmlu/full/ja/_ja_template_yaml     |  16 ++
 .../global_mmlu_full_ja_abstract_algebra.yaml |   5 +
 .../full/ja/global_mmlu_full_ja_anatomy.yaml  |   5 +
 .../ja/global_mmlu_full_ja_astronomy.yaml     |   5 +
 .../global_mmlu_full_ja_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_ja_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_ja_college_biology.yaml  |   5 +
 ...global_mmlu_full_ja_college_chemistry.yaml |   5 +
 ...mmlu_full_ja_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_ja_college_mathematics.yaml |   5 +
 .../global_mmlu_full_ja_college_medicine.yaml |   5 +
 .../global_mmlu_full_ja_college_physics.yaml  |   5 +
 ...global_mmlu_full_ja_computer_security.yaml |   5 +
 ...lobal_mmlu_full_ja_conceptual_physics.yaml |   5 +
 .../ja/global_mmlu_full_ja_econometrics.yaml  |   5 +
 ...l_mmlu_full_ja_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_ja_elementary_mathematics.yaml |   5 +
 .../ja/global_mmlu_full_ja_formal_logic.yaml  |   5 +
 .../ja/global_mmlu_full_ja_global_facts.yaml  |   5 +
 ...obal_mmlu_full_ja_high_school_biology.yaml |   5 +
 ...al_mmlu_full_ja_high_school_chemistry.yaml |   5 +
 ..._full_ja_high_school_computer_science.yaml |   5 +
 ..._full_ja_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_ja_high_school_geography.yaml |   5 +
 ...a_high_school_government_and_politics.yaml |   5 +
 ...lu_full_ja_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_ja_high_school_mathematics.yaml |   5 +
 ...lu_full_ja_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_ja_high_school_physics.yaml |   5 +
 ...l_mmlu_full_ja_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_ja_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_ja_high_school_us_history.yaml |   5 +
 ...mlu_full_ja_high_school_world_history.yaml |   5 +
 .../ja/global_mmlu_full_ja_human_aging.yaml   |   5 +
 .../global_mmlu_full_ja_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_ja_international_law.yaml |   5 +
 .../ja/global_mmlu_full_ja_jurisprudence.yaml |   5 +
 ...global_mmlu_full_ja_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_ja_machine_learning.yaml |   5 +
 .../ja/global_mmlu_full_ja_management.yaml    |   5 +
 .../ja/global_mmlu_full_ja_marketing.yaml     |   5 +
 .../global_mmlu_full_ja_medical_genetics.yaml |   5 +
 .../ja/global_mmlu_full_ja_miscellaneous.yaml |   5 +
 .../global_mmlu_full_ja_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_ja_moral_scenarios.yaml  |   5 +
 .../ja/global_mmlu_full_ja_nutrition.yaml     |   5 +
 .../ja/global_mmlu_full_ja_philosophy.yaml    |   5 +
 .../ja/global_mmlu_full_ja_prehistory.yaml    |   5 +
 ..._mmlu_full_ja_professional_accounting.yaml |   5 +
 .../global_mmlu_full_ja_professional_law.yaml |   5 +
 ...al_mmlu_full_ja_professional_medicine.yaml |   5 +
 ..._mmlu_full_ja_professional_psychology.yaml |   5 +
 .../global_mmlu_full_ja_public_relations.yaml |   5 +
 .../global_mmlu_full_ja_security_studies.yaml |   5 +
 .../ja/global_mmlu_full_ja_sociology.yaml     |   5 +
 ...global_mmlu_full_ja_us_foreign_policy.yaml |   5 +
 .../full/ja/global_mmlu_full_ja_virology.yaml |   5 +
 .../global_mmlu_full_ja_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/ja/utils.py  |  73 +++++
 .../full/ko/_global_mmlu_full_ko.yaml         |  11 +
 .../ko/_global_mmlu_full_ko_humanities.yaml   |   8 +
 .../full/ko/_global_mmlu_full_ko_other.yaml   |   8 +
 .../_global_mmlu_full_ko_social_sciences.yaml |   8 +
 .../full/ko/_global_mmlu_full_ko_stem.yaml    |   8 +
 .../global_mmlu/full/ko/_ko_template_yaml     |  16 ++
 .../global_mmlu_full_ko_abstract_algebra.yaml |   5 +
 .../full/ko/global_mmlu_full_ko_anatomy.yaml  |   5 +
 .../ko/global_mmlu_full_ko_astronomy.yaml     |   5 +
 .../global_mmlu_full_ko_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_ko_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_ko_college_biology.yaml  |   5 +
 ...global_mmlu_full_ko_college_chemistry.yaml |   5 +
 ...mmlu_full_ko_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_ko_college_mathematics.yaml |   5 +
 .../global_mmlu_full_ko_college_medicine.yaml |   5 +
 .../global_mmlu_full_ko_college_physics.yaml  |   5 +
 ...global_mmlu_full_ko_computer_security.yaml |   5 +
 ...lobal_mmlu_full_ko_conceptual_physics.yaml |   5 +
 .../ko/global_mmlu_full_ko_econometrics.yaml  |   5 +
 ...l_mmlu_full_ko_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_ko_elementary_mathematics.yaml |   5 +
 .../ko/global_mmlu_full_ko_formal_logic.yaml  |   5 +
 .../ko/global_mmlu_full_ko_global_facts.yaml  |   5 +
 ...obal_mmlu_full_ko_high_school_biology.yaml |   5 +
 ...al_mmlu_full_ko_high_school_chemistry.yaml |   5 +
 ..._full_ko_high_school_computer_science.yaml |   5 +
 ..._full_ko_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_ko_high_school_geography.yaml |   5 +
 ...o_high_school_government_and_politics.yaml |   5 +
 ...lu_full_ko_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_ko_high_school_mathematics.yaml |   5 +
 ...lu_full_ko_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_ko_high_school_physics.yaml |   5 +
 ...l_mmlu_full_ko_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_ko_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_ko_high_school_us_history.yaml |   5 +
 ...mlu_full_ko_high_school_world_history.yaml |   5 +
 .../ko/global_mmlu_full_ko_human_aging.yaml   |   5 +
 .../global_mmlu_full_ko_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_ko_international_law.yaml |   5 +
 .../ko/global_mmlu_full_ko_jurisprudence.yaml |   5 +
 ...global_mmlu_full_ko_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_ko_machine_learning.yaml |   5 +
 .../ko/global_mmlu_full_ko_management.yaml    |   5 +
 .../ko/global_mmlu_full_ko_marketing.yaml     |   5 +
 .../global_mmlu_full_ko_medical_genetics.yaml |   5 +
 .../ko/global_mmlu_full_ko_miscellaneous.yaml |   5 +
 .../global_mmlu_full_ko_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_ko_moral_scenarios.yaml  |   5 +
 .../ko/global_mmlu_full_ko_nutrition.yaml     |   5 +
 .../ko/global_mmlu_full_ko_philosophy.yaml    |   5 +
 .../ko/global_mmlu_full_ko_prehistory.yaml    |   5 +
 ..._mmlu_full_ko_professional_accounting.yaml |   5 +
 .../global_mmlu_full_ko_professional_law.yaml |   5 +
 ...al_mmlu_full_ko_professional_medicine.yaml |   5 +
 ..._mmlu_full_ko_professional_psychology.yaml |   5 +
 .../global_mmlu_full_ko_public_relations.yaml |   5 +
 .../global_mmlu_full_ko_security_studies.yaml |   5 +
 .../ko/global_mmlu_full_ko_sociology.yaml     |   5 +
 ...global_mmlu_full_ko_us_foreign_policy.yaml |   5 +
 .../full/ko/global_mmlu_full_ko_virology.yaml |   5 +
 .../global_mmlu_full_ko_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/ko/utils.py  |  73 +++++
 .../full/ky/_global_mmlu_full_ky.yaml         |  11 +
 .../ky/_global_mmlu_full_ky_humanities.yaml   |   8 +
 .../full/ky/_global_mmlu_full_ky_other.yaml   |   8 +
 .../_global_mmlu_full_ky_social_sciences.yaml |   8 +
 .../full/ky/_global_mmlu_full_ky_stem.yaml    |   8 +
 .../global_mmlu/full/ky/_ky_template_yaml     |  16 ++
 .../global_mmlu_full_ky_abstract_algebra.yaml |   5 +
 .../full/ky/global_mmlu_full_ky_anatomy.yaml  |   5 +
 .../ky/global_mmlu_full_ky_astronomy.yaml     |   5 +
 .../global_mmlu_full_ky_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_ky_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_ky_college_biology.yaml  |   5 +
 ...global_mmlu_full_ky_college_chemistry.yaml |   5 +
 ...mmlu_full_ky_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_ky_college_mathematics.yaml |   5 +
 .../global_mmlu_full_ky_college_medicine.yaml |   5 +
 .../global_mmlu_full_ky_college_physics.yaml  |   5 +
 ...global_mmlu_full_ky_computer_security.yaml |   5 +
 ...lobal_mmlu_full_ky_conceptual_physics.yaml |   5 +
 .../ky/global_mmlu_full_ky_econometrics.yaml  |   5 +
 ...l_mmlu_full_ky_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_ky_elementary_mathematics.yaml |   5 +
 .../ky/global_mmlu_full_ky_formal_logic.yaml  |   5 +
 .../ky/global_mmlu_full_ky_global_facts.yaml  |   5 +
 ...obal_mmlu_full_ky_high_school_biology.yaml |   5 +
 ...al_mmlu_full_ky_high_school_chemistry.yaml |   5 +
 ..._full_ky_high_school_computer_science.yaml |   5 +
 ..._full_ky_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_ky_high_school_geography.yaml |   5 +
 ...y_high_school_government_and_politics.yaml |   5 +
 ...lu_full_ky_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_ky_high_school_mathematics.yaml |   5 +
 ...lu_full_ky_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_ky_high_school_physics.yaml |   5 +
 ...l_mmlu_full_ky_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_ky_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_ky_high_school_us_history.yaml |   5 +
 ...mlu_full_ky_high_school_world_history.yaml |   5 +
 .../ky/global_mmlu_full_ky_human_aging.yaml   |   5 +
 .../global_mmlu_full_ky_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_ky_international_law.yaml |   5 +
 .../ky/global_mmlu_full_ky_jurisprudence.yaml |   5 +
 ...global_mmlu_full_ky_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_ky_machine_learning.yaml |   5 +
 .../ky/global_mmlu_full_ky_management.yaml    |   5 +
 .../ky/global_mmlu_full_ky_marketing.yaml     |   5 +
 .../global_mmlu_full_ky_medical_genetics.yaml |   5 +
 .../ky/global_mmlu_full_ky_miscellaneous.yaml |   5 +
 .../global_mmlu_full_ky_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_ky_moral_scenarios.yaml  |   5 +
 .../ky/global_mmlu_full_ky_nutrition.yaml     |   5 +
 .../ky/global_mmlu_full_ky_philosophy.yaml    |   5 +
 .../ky/global_mmlu_full_ky_prehistory.yaml    |   5 +
 ..._mmlu_full_ky_professional_accounting.yaml |   5 +
 .../global_mmlu_full_ky_professional_law.yaml |   5 +
 ...al_mmlu_full_ky_professional_medicine.yaml |   5 +
 ..._mmlu_full_ky_professional_psychology.yaml |   5 +
 .../global_mmlu_full_ky_public_relations.yaml |   5 +
 .../global_mmlu_full_ky_security_studies.yaml |   5 +
 .../ky/global_mmlu_full_ky_sociology.yaml     |   5 +
 ...global_mmlu_full_ky_us_foreign_policy.yaml |   5 +
 .../full/ky/global_mmlu_full_ky_virology.yaml |   5 +
 .../global_mmlu_full_ky_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/ky/utils.py  |  73 +++++
 .../full/lt/_global_mmlu_full_lt.yaml         |  11 +
 .../lt/_global_mmlu_full_lt_humanities.yaml   |   8 +
 .../full/lt/_global_mmlu_full_lt_other.yaml   |   8 +
 .../_global_mmlu_full_lt_social_sciences.yaml |   8 +
 .../full/lt/_global_mmlu_full_lt_stem.yaml    |   8 +
 .../global_mmlu/full/lt/_lt_template_yaml     |  16 ++
 .../global_mmlu_full_lt_abstract_algebra.yaml |   5 +
 .../full/lt/global_mmlu_full_lt_anatomy.yaml  |   5 +
 .../lt/global_mmlu_full_lt_astronomy.yaml     |   5 +
 .../global_mmlu_full_lt_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_lt_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_lt_college_biology.yaml  |   5 +
 ...global_mmlu_full_lt_college_chemistry.yaml |   5 +
 ...mmlu_full_lt_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_lt_college_mathematics.yaml |   5 +
 .../global_mmlu_full_lt_college_medicine.yaml |   5 +
 .../global_mmlu_full_lt_college_physics.yaml  |   5 +
 ...global_mmlu_full_lt_computer_security.yaml |   5 +
 ...lobal_mmlu_full_lt_conceptual_physics.yaml |   5 +
 .../lt/global_mmlu_full_lt_econometrics.yaml  |   5 +
 ...l_mmlu_full_lt_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_lt_elementary_mathematics.yaml |   5 +
 .../lt/global_mmlu_full_lt_formal_logic.yaml  |   5 +
 .../lt/global_mmlu_full_lt_global_facts.yaml  |   5 +
 ...obal_mmlu_full_lt_high_school_biology.yaml |   5 +
 ...al_mmlu_full_lt_high_school_chemistry.yaml |   5 +
 ..._full_lt_high_school_computer_science.yaml |   5 +
 ..._full_lt_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_lt_high_school_geography.yaml |   5 +
 ...t_high_school_government_and_politics.yaml |   5 +
 ...lu_full_lt_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_lt_high_school_mathematics.yaml |   5 +
 ...lu_full_lt_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_lt_high_school_physics.yaml |   5 +
 ...l_mmlu_full_lt_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_lt_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_lt_high_school_us_history.yaml |   5 +
 ...mlu_full_lt_high_school_world_history.yaml |   5 +
 .../lt/global_mmlu_full_lt_human_aging.yaml   |   5 +
 .../global_mmlu_full_lt_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_lt_international_law.yaml |   5 +
 .../lt/global_mmlu_full_lt_jurisprudence.yaml |   5 +
 ...global_mmlu_full_lt_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_lt_machine_learning.yaml |   5 +
 .../lt/global_mmlu_full_lt_management.yaml    |   5 +
 .../lt/global_mmlu_full_lt_marketing.yaml     |   5 +
 .../global_mmlu_full_lt_medical_genetics.yaml |   5 +
 .../lt/global_mmlu_full_lt_miscellaneous.yaml |   5 +
 .../global_mmlu_full_lt_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_lt_moral_scenarios.yaml  |   5 +
 .../lt/global_mmlu_full_lt_nutrition.yaml     |   5 +
 .../lt/global_mmlu_full_lt_philosophy.yaml    |   5 +
 .../lt/global_mmlu_full_lt_prehistory.yaml    |   5 +
 ..._mmlu_full_lt_professional_accounting.yaml |   5 +
 .../global_mmlu_full_lt_professional_law.yaml |   5 +
 ...al_mmlu_full_lt_professional_medicine.yaml |   5 +
 ..._mmlu_full_lt_professional_psychology.yaml |   5 +
 .../global_mmlu_full_lt_public_relations.yaml |   5 +
 .../global_mmlu_full_lt_security_studies.yaml |   5 +
 .../lt/global_mmlu_full_lt_sociology.yaml     |   5 +
 ...global_mmlu_full_lt_us_foreign_policy.yaml |   5 +
 .../full/lt/global_mmlu_full_lt_virology.yaml |   5 +
 .../global_mmlu_full_lt_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/lt/utils.py  |  73 +++++
 .../full/mg/_global_mmlu_full_mg.yaml         |  11 +
 .../mg/_global_mmlu_full_mg_humanities.yaml   |   8 +
 .../full/mg/_global_mmlu_full_mg_other.yaml   |   8 +
 .../_global_mmlu_full_mg_social_sciences.yaml |   8 +
 .../full/mg/_global_mmlu_full_mg_stem.yaml    |   8 +
 .../global_mmlu/full/mg/_mg_template_yaml     |  16 ++
 .../global_mmlu_full_mg_abstract_algebra.yaml |   5 +
 .../full/mg/global_mmlu_full_mg_anatomy.yaml  |   5 +
 .../mg/global_mmlu_full_mg_astronomy.yaml     |   5 +
 .../global_mmlu_full_mg_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_mg_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_mg_college_biology.yaml  |   5 +
 ...global_mmlu_full_mg_college_chemistry.yaml |   5 +
 ...mmlu_full_mg_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_mg_college_mathematics.yaml |   5 +
 .../global_mmlu_full_mg_college_medicine.yaml |   5 +
 .../global_mmlu_full_mg_college_physics.yaml  |   5 +
 ...global_mmlu_full_mg_computer_security.yaml |   5 +
 ...lobal_mmlu_full_mg_conceptual_physics.yaml |   5 +
 .../mg/global_mmlu_full_mg_econometrics.yaml  |   5 +
 ...l_mmlu_full_mg_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_mg_elementary_mathematics.yaml |   5 +
 .../mg/global_mmlu_full_mg_formal_logic.yaml  |   5 +
 .../mg/global_mmlu_full_mg_global_facts.yaml  |   5 +
 ...obal_mmlu_full_mg_high_school_biology.yaml |   5 +
 ...al_mmlu_full_mg_high_school_chemistry.yaml |   5 +
 ..._full_mg_high_school_computer_science.yaml |   5 +
 ..._full_mg_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_mg_high_school_geography.yaml |   5 +
 ...g_high_school_government_and_politics.yaml |   5 +
 ...lu_full_mg_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_mg_high_school_mathematics.yaml |   5 +
 ...lu_full_mg_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_mg_high_school_physics.yaml |   5 +
 ...l_mmlu_full_mg_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_mg_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_mg_high_school_us_history.yaml |   5 +
 ...mlu_full_mg_high_school_world_history.yaml |   5 +
 .../mg/global_mmlu_full_mg_human_aging.yaml   |   5 +
 .../global_mmlu_full_mg_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_mg_international_law.yaml |   5 +
 .../mg/global_mmlu_full_mg_jurisprudence.yaml |   5 +
 ...global_mmlu_full_mg_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_mg_machine_learning.yaml |   5 +
 .../mg/global_mmlu_full_mg_management.yaml    |   5 +
 .../mg/global_mmlu_full_mg_marketing.yaml     |   5 +
 .../global_mmlu_full_mg_medical_genetics.yaml |   5 +
 .../mg/global_mmlu_full_mg_miscellaneous.yaml |   5 +
 .../global_mmlu_full_mg_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_mg_moral_scenarios.yaml  |   5 +
 .../mg/global_mmlu_full_mg_nutrition.yaml     |   5 +
 .../mg/global_mmlu_full_mg_philosophy.yaml    |   5 +
 .../mg/global_mmlu_full_mg_prehistory.yaml    |   5 +
 ..._mmlu_full_mg_professional_accounting.yaml |   5 +
 .../global_mmlu_full_mg_professional_law.yaml |   5 +
 ...al_mmlu_full_mg_professional_medicine.yaml |   5 +
 ..._mmlu_full_mg_professional_psychology.yaml |   5 +
 .../global_mmlu_full_mg_public_relations.yaml |   5 +
 .../global_mmlu_full_mg_security_studies.yaml |   5 +
 .../mg/global_mmlu_full_mg_sociology.yaml     |   5 +
 ...global_mmlu_full_mg_us_foreign_policy.yaml |   5 +
 .../full/mg/global_mmlu_full_mg_virology.yaml |   5 +
 .../global_mmlu_full_mg_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/mg/utils.py  |  73 +++++
 .../full/ms/_global_mmlu_full_ms.yaml         |  11 +
 .../ms/_global_mmlu_full_ms_humanities.yaml   |   8 +
 .../full/ms/_global_mmlu_full_ms_other.yaml   |   8 +
 .../_global_mmlu_full_ms_social_sciences.yaml |   8 +
 .../full/ms/_global_mmlu_full_ms_stem.yaml    |   8 +
 .../global_mmlu/full/ms/_ms_template_yaml     |  16 ++
 .../global_mmlu_full_ms_abstract_algebra.yaml |   5 +
 .../full/ms/global_mmlu_full_ms_anatomy.yaml  |   5 +
 .../ms/global_mmlu_full_ms_astronomy.yaml     |   5 +
 .../global_mmlu_full_ms_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_ms_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_ms_college_biology.yaml  |   5 +
 ...global_mmlu_full_ms_college_chemistry.yaml |   5 +
 ...mmlu_full_ms_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_ms_college_mathematics.yaml |   5 +
 .../global_mmlu_full_ms_college_medicine.yaml |   5 +
 .../global_mmlu_full_ms_college_physics.yaml  |   5 +
 ...global_mmlu_full_ms_computer_security.yaml |   5 +
 ...lobal_mmlu_full_ms_conceptual_physics.yaml |   5 +
 .../ms/global_mmlu_full_ms_econometrics.yaml  |   5 +
 ...l_mmlu_full_ms_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_ms_elementary_mathematics.yaml |   5 +
 .../ms/global_mmlu_full_ms_formal_logic.yaml  |   5 +
 .../ms/global_mmlu_full_ms_global_facts.yaml  |   5 +
 ...obal_mmlu_full_ms_high_school_biology.yaml |   5 +
 ...al_mmlu_full_ms_high_school_chemistry.yaml |   5 +
 ..._full_ms_high_school_computer_science.yaml |   5 +
 ..._full_ms_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_ms_high_school_geography.yaml |   5 +
 ...s_high_school_government_and_politics.yaml |   5 +
 ...lu_full_ms_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_ms_high_school_mathematics.yaml |   5 +
 ...lu_full_ms_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_ms_high_school_physics.yaml |   5 +
 ...l_mmlu_full_ms_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_ms_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_ms_high_school_us_history.yaml |   5 +
 ...mlu_full_ms_high_school_world_history.yaml |   5 +
 .../ms/global_mmlu_full_ms_human_aging.yaml   |   5 +
 .../global_mmlu_full_ms_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_ms_international_law.yaml |   5 +
 .../ms/global_mmlu_full_ms_jurisprudence.yaml |   5 +
 ...global_mmlu_full_ms_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_ms_machine_learning.yaml |   5 +
 .../ms/global_mmlu_full_ms_management.yaml    |   5 +
 .../ms/global_mmlu_full_ms_marketing.yaml     |   5 +
 .../global_mmlu_full_ms_medical_genetics.yaml |   5 +
 .../ms/global_mmlu_full_ms_miscellaneous.yaml |   5 +
 .../global_mmlu_full_ms_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_ms_moral_scenarios.yaml  |   5 +
 .../ms/global_mmlu_full_ms_nutrition.yaml     |   5 +
 .../ms/global_mmlu_full_ms_philosophy.yaml    |   5 +
 .../ms/global_mmlu_full_ms_prehistory.yaml    |   5 +
 ..._mmlu_full_ms_professional_accounting.yaml |   5 +
 .../global_mmlu_full_ms_professional_law.yaml |   5 +
 ...al_mmlu_full_ms_professional_medicine.yaml |   5 +
 ..._mmlu_full_ms_professional_psychology.yaml |   5 +
 .../global_mmlu_full_ms_public_relations.yaml |   5 +
 .../global_mmlu_full_ms_security_studies.yaml |   5 +
 .../ms/global_mmlu_full_ms_sociology.yaml     |   5 +
 ...global_mmlu_full_ms_us_foreign_policy.yaml |   5 +
 .../full/ms/global_mmlu_full_ms_virology.yaml |   5 +
 .../global_mmlu_full_ms_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/ms/utils.py  |  73 +++++
 .../full/ne/_global_mmlu_full_ne.yaml         |  11 +
 .../ne/_global_mmlu_full_ne_humanities.yaml   |   8 +
 .../full/ne/_global_mmlu_full_ne_other.yaml   |   8 +
 .../_global_mmlu_full_ne_social_sciences.yaml |   8 +
 .../full/ne/_global_mmlu_full_ne_stem.yaml    |   8 +
 .../global_mmlu/full/ne/_ne_template_yaml     |  16 ++
 .../global_mmlu_full_ne_abstract_algebra.yaml |   5 +
 .../full/ne/global_mmlu_full_ne_anatomy.yaml  |   5 +
 .../ne/global_mmlu_full_ne_astronomy.yaml     |   5 +
 .../global_mmlu_full_ne_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_ne_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_ne_college_biology.yaml  |   5 +
 ...global_mmlu_full_ne_college_chemistry.yaml |   5 +
 ...mmlu_full_ne_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_ne_college_mathematics.yaml |   5 +
 .../global_mmlu_full_ne_college_medicine.yaml |   5 +
 .../global_mmlu_full_ne_college_physics.yaml  |   5 +
 ...global_mmlu_full_ne_computer_security.yaml |   5 +
 ...lobal_mmlu_full_ne_conceptual_physics.yaml |   5 +
 .../ne/global_mmlu_full_ne_econometrics.yaml  |   5 +
 ...l_mmlu_full_ne_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_ne_elementary_mathematics.yaml |   5 +
 .../ne/global_mmlu_full_ne_formal_logic.yaml  |   5 +
 .../ne/global_mmlu_full_ne_global_facts.yaml  |   5 +
 ...obal_mmlu_full_ne_high_school_biology.yaml |   5 +
 ...al_mmlu_full_ne_high_school_chemistry.yaml |   5 +
 ..._full_ne_high_school_computer_science.yaml |   5 +
 ..._full_ne_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_ne_high_school_geography.yaml |   5 +
 ...e_high_school_government_and_politics.yaml |   5 +
 ...lu_full_ne_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_ne_high_school_mathematics.yaml |   5 +
 ...lu_full_ne_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_ne_high_school_physics.yaml |   5 +
 ...l_mmlu_full_ne_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_ne_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_ne_high_school_us_history.yaml |   5 +
 ...mlu_full_ne_high_school_world_history.yaml |   5 +
 .../ne/global_mmlu_full_ne_human_aging.yaml   |   5 +
 .../global_mmlu_full_ne_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_ne_international_law.yaml |   5 +
 .../ne/global_mmlu_full_ne_jurisprudence.yaml |   5 +
 ...global_mmlu_full_ne_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_ne_machine_learning.yaml |   5 +
 .../ne/global_mmlu_full_ne_management.yaml    |   5 +
 .../ne/global_mmlu_full_ne_marketing.yaml     |   5 +
 .../global_mmlu_full_ne_medical_genetics.yaml |   5 +
 .../ne/global_mmlu_full_ne_miscellaneous.yaml |   5 +
 .../global_mmlu_full_ne_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_ne_moral_scenarios.yaml  |   5 +
 .../ne/global_mmlu_full_ne_nutrition.yaml     |   5 +
 .../ne/global_mmlu_full_ne_philosophy.yaml    |   5 +
 .../ne/global_mmlu_full_ne_prehistory.yaml    |   5 +
 ..._mmlu_full_ne_professional_accounting.yaml |   5 +
 .../global_mmlu_full_ne_professional_law.yaml |   5 +
 ...al_mmlu_full_ne_professional_medicine.yaml |   5 +
 ..._mmlu_full_ne_professional_psychology.yaml |   5 +
 .../global_mmlu_full_ne_public_relations.yaml |   5 +
 .../global_mmlu_full_ne_security_studies.yaml |   5 +
 .../ne/global_mmlu_full_ne_sociology.yaml     |   5 +
 ...global_mmlu_full_ne_us_foreign_policy.yaml |   5 +
 .../full/ne/global_mmlu_full_ne_virology.yaml |   5 +
 .../global_mmlu_full_ne_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/ne/utils.py  |  73 +++++
 .../full/nl/_global_mmlu_full_nl.yaml         |  11 +
 .../nl/_global_mmlu_full_nl_humanities.yaml   |   8 +
 .../full/nl/_global_mmlu_full_nl_other.yaml   |   8 +
 .../_global_mmlu_full_nl_social_sciences.yaml |   8 +
 .../full/nl/_global_mmlu_full_nl_stem.yaml    |   8 +
 .../global_mmlu/full/nl/_nl_template_yaml     |  16 ++
 .../global_mmlu_full_nl_abstract_algebra.yaml |   5 +
 .../full/nl/global_mmlu_full_nl_anatomy.yaml  |   5 +
 .../nl/global_mmlu_full_nl_astronomy.yaml     |   5 +
 .../global_mmlu_full_nl_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_nl_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_nl_college_biology.yaml  |   5 +
 ...global_mmlu_full_nl_college_chemistry.yaml |   5 +
 ...mmlu_full_nl_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_nl_college_mathematics.yaml |   5 +
 .../global_mmlu_full_nl_college_medicine.yaml |   5 +
 .../global_mmlu_full_nl_college_physics.yaml  |   5 +
 ...global_mmlu_full_nl_computer_security.yaml |   5 +
 ...lobal_mmlu_full_nl_conceptual_physics.yaml |   5 +
 .../nl/global_mmlu_full_nl_econometrics.yaml  |   5 +
 ...l_mmlu_full_nl_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_nl_elementary_mathematics.yaml |   5 +
 .../nl/global_mmlu_full_nl_formal_logic.yaml  |   5 +
 .../nl/global_mmlu_full_nl_global_facts.yaml  |   5 +
 ...obal_mmlu_full_nl_high_school_biology.yaml |   5 +
 ...al_mmlu_full_nl_high_school_chemistry.yaml |   5 +
 ..._full_nl_high_school_computer_science.yaml |   5 +
 ..._full_nl_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_nl_high_school_geography.yaml |   5 +
 ...l_high_school_government_and_politics.yaml |   5 +
 ...lu_full_nl_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_nl_high_school_mathematics.yaml |   5 +
 ...lu_full_nl_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_nl_high_school_physics.yaml |   5 +
 ...l_mmlu_full_nl_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_nl_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_nl_high_school_us_history.yaml |   5 +
 ...mlu_full_nl_high_school_world_history.yaml |   5 +
 .../nl/global_mmlu_full_nl_human_aging.yaml   |   5 +
 .../global_mmlu_full_nl_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_nl_international_law.yaml |   5 +
 .../nl/global_mmlu_full_nl_jurisprudence.yaml |   5 +
 ...global_mmlu_full_nl_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_nl_machine_learning.yaml |   5 +
 .../nl/global_mmlu_full_nl_management.yaml    |   5 +
 .../nl/global_mmlu_full_nl_marketing.yaml     |   5 +
 .../global_mmlu_full_nl_medical_genetics.yaml |   5 +
 .../nl/global_mmlu_full_nl_miscellaneous.yaml |   5 +
 .../global_mmlu_full_nl_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_nl_moral_scenarios.yaml  |   5 +
 .../nl/global_mmlu_full_nl_nutrition.yaml     |   5 +
 .../nl/global_mmlu_full_nl_philosophy.yaml    |   5 +
 .../nl/global_mmlu_full_nl_prehistory.yaml    |   5 +
 ..._mmlu_full_nl_professional_accounting.yaml |   5 +
 .../global_mmlu_full_nl_professional_law.yaml |   5 +
 ...al_mmlu_full_nl_professional_medicine.yaml |   5 +
 ..._mmlu_full_nl_professional_psychology.yaml |   5 +
 .../global_mmlu_full_nl_public_relations.yaml |   5 +
 .../global_mmlu_full_nl_security_studies.yaml |   5 +
 .../nl/global_mmlu_full_nl_sociology.yaml     |   5 +
 ...global_mmlu_full_nl_us_foreign_policy.yaml |   5 +
 .../full/nl/global_mmlu_full_nl_virology.yaml |   5 +
 .../global_mmlu_full_nl_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/nl/utils.py  |  73 +++++
 .../full/ny/_global_mmlu_full_ny.yaml         |  11 +
 .../ny/_global_mmlu_full_ny_humanities.yaml   |   8 +
 .../full/ny/_global_mmlu_full_ny_other.yaml   |   8 +
 .../_global_mmlu_full_ny_social_sciences.yaml |   8 +
 .../full/ny/_global_mmlu_full_ny_stem.yaml    |   8 +
 .../global_mmlu/full/ny/_ny_template_yaml     |  16 ++
 .../global_mmlu_full_ny_abstract_algebra.yaml |   5 +
 .../full/ny/global_mmlu_full_ny_anatomy.yaml  |   5 +
 .../ny/global_mmlu_full_ny_astronomy.yaml     |   5 +
 .../global_mmlu_full_ny_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_ny_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_ny_college_biology.yaml  |   5 +
 ...global_mmlu_full_ny_college_chemistry.yaml |   5 +
 ...mmlu_full_ny_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_ny_college_mathematics.yaml |   5 +
 .../global_mmlu_full_ny_college_medicine.yaml |   5 +
 .../global_mmlu_full_ny_college_physics.yaml  |   5 +
 ...global_mmlu_full_ny_computer_security.yaml |   5 +
 ...lobal_mmlu_full_ny_conceptual_physics.yaml |   5 +
 .../ny/global_mmlu_full_ny_econometrics.yaml  |   5 +
 ...l_mmlu_full_ny_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_ny_elementary_mathematics.yaml |   5 +
 .../ny/global_mmlu_full_ny_formal_logic.yaml  |   5 +
 .../ny/global_mmlu_full_ny_global_facts.yaml  |   5 +
 ...obal_mmlu_full_ny_high_school_biology.yaml |   5 +
 ...al_mmlu_full_ny_high_school_chemistry.yaml |   5 +
 ..._full_ny_high_school_computer_science.yaml |   5 +
 ..._full_ny_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_ny_high_school_geography.yaml |   5 +
 ...y_high_school_government_and_politics.yaml |   5 +
 ...lu_full_ny_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_ny_high_school_mathematics.yaml |   5 +
 ...lu_full_ny_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_ny_high_school_physics.yaml |   5 +
 ...l_mmlu_full_ny_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_ny_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_ny_high_school_us_history.yaml |   5 +
 ...mlu_full_ny_high_school_world_history.yaml |   5 +
 .../ny/global_mmlu_full_ny_human_aging.yaml   |   5 +
 .../global_mmlu_full_ny_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_ny_international_law.yaml |   5 +
 .../ny/global_mmlu_full_ny_jurisprudence.yaml |   5 +
 ...global_mmlu_full_ny_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_ny_machine_learning.yaml |   5 +
 .../ny/global_mmlu_full_ny_management.yaml    |   5 +
 .../ny/global_mmlu_full_ny_marketing.yaml     |   5 +
 .../global_mmlu_full_ny_medical_genetics.yaml |   5 +
 .../ny/global_mmlu_full_ny_miscellaneous.yaml |   5 +
 .../global_mmlu_full_ny_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_ny_moral_scenarios.yaml  |   5 +
 .../ny/global_mmlu_full_ny_nutrition.yaml     |   5 +
 .../ny/global_mmlu_full_ny_philosophy.yaml    |   5 +
 .../ny/global_mmlu_full_ny_prehistory.yaml    |   5 +
 ..._mmlu_full_ny_professional_accounting.yaml |   5 +
 .../global_mmlu_full_ny_professional_law.yaml |   5 +
 ...al_mmlu_full_ny_professional_medicine.yaml |   5 +
 ..._mmlu_full_ny_professional_psychology.yaml |   5 +
 .../global_mmlu_full_ny_public_relations.yaml |   5 +
 .../global_mmlu_full_ny_security_studies.yaml |   5 +
 .../ny/global_mmlu_full_ny_sociology.yaml     |   5 +
 ...global_mmlu_full_ny_us_foreign_policy.yaml |   5 +
 .../full/ny/global_mmlu_full_ny_virology.yaml |   5 +
 .../global_mmlu_full_ny_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/ny/utils.py  |  73 +++++
 .../full/pl/_global_mmlu_full_pl.yaml         |  11 +
 .../pl/_global_mmlu_full_pl_humanities.yaml   |   8 +
 .../full/pl/_global_mmlu_full_pl_other.yaml   |   8 +
 .../_global_mmlu_full_pl_social_sciences.yaml |   8 +
 .../full/pl/_global_mmlu_full_pl_stem.yaml    |   8 +
 .../global_mmlu/full/pl/_pl_template_yaml     |  16 ++
 .../global_mmlu_full_pl_abstract_algebra.yaml |   5 +
 .../full/pl/global_mmlu_full_pl_anatomy.yaml  |   5 +
 .../pl/global_mmlu_full_pl_astronomy.yaml     |   5 +
 .../global_mmlu_full_pl_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_pl_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_pl_college_biology.yaml  |   5 +
 ...global_mmlu_full_pl_college_chemistry.yaml |   5 +
 ...mmlu_full_pl_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_pl_college_mathematics.yaml |   5 +
 .../global_mmlu_full_pl_college_medicine.yaml |   5 +
 .../global_mmlu_full_pl_college_physics.yaml  |   5 +
 ...global_mmlu_full_pl_computer_security.yaml |   5 +
 ...lobal_mmlu_full_pl_conceptual_physics.yaml |   5 +
 .../pl/global_mmlu_full_pl_econometrics.yaml  |   5 +
 ...l_mmlu_full_pl_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_pl_elementary_mathematics.yaml |   5 +
 .../pl/global_mmlu_full_pl_formal_logic.yaml  |   5 +
 .../pl/global_mmlu_full_pl_global_facts.yaml  |   5 +
 ...obal_mmlu_full_pl_high_school_biology.yaml |   5 +
 ...al_mmlu_full_pl_high_school_chemistry.yaml |   5 +
 ..._full_pl_high_school_computer_science.yaml |   5 +
 ..._full_pl_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_pl_high_school_geography.yaml |   5 +
 ...l_high_school_government_and_politics.yaml |   5 +
 ...lu_full_pl_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_pl_high_school_mathematics.yaml |   5 +
 ...lu_full_pl_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_pl_high_school_physics.yaml |   5 +
 ...l_mmlu_full_pl_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_pl_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_pl_high_school_us_history.yaml |   5 +
 ...mlu_full_pl_high_school_world_history.yaml |   5 +
 .../pl/global_mmlu_full_pl_human_aging.yaml   |   5 +
 .../global_mmlu_full_pl_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_pl_international_law.yaml |   5 +
 .../pl/global_mmlu_full_pl_jurisprudence.yaml |   5 +
 ...global_mmlu_full_pl_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_pl_machine_learning.yaml |   5 +
 .../pl/global_mmlu_full_pl_management.yaml    |   5 +
 .../pl/global_mmlu_full_pl_marketing.yaml     |   5 +
 .../global_mmlu_full_pl_medical_genetics.yaml |   5 +
 .../pl/global_mmlu_full_pl_miscellaneous.yaml |   5 +
 .../global_mmlu_full_pl_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_pl_moral_scenarios.yaml  |   5 +
 .../pl/global_mmlu_full_pl_nutrition.yaml     |   5 +
 .../pl/global_mmlu_full_pl_philosophy.yaml    |   5 +
 .../pl/global_mmlu_full_pl_prehistory.yaml    |   5 +
 ..._mmlu_full_pl_professional_accounting.yaml |   5 +
 .../global_mmlu_full_pl_professional_law.yaml |   5 +
 ...al_mmlu_full_pl_professional_medicine.yaml |   5 +
 ..._mmlu_full_pl_professional_psychology.yaml |   5 +
 .../global_mmlu_full_pl_public_relations.yaml |   5 +
 .../global_mmlu_full_pl_security_studies.yaml |   5 +
 .../pl/global_mmlu_full_pl_sociology.yaml     |   5 +
 ...global_mmlu_full_pl_us_foreign_policy.yaml |   5 +
 .../full/pl/global_mmlu_full_pl_virology.yaml |   5 +
 .../global_mmlu_full_pl_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/pl/utils.py  |  73 +++++
 .../full/pt/_global_mmlu_full_pt.yaml         |  11 +
 .../pt/_global_mmlu_full_pt_humanities.yaml   |   8 +
 .../full/pt/_global_mmlu_full_pt_other.yaml   |   8 +
 .../_global_mmlu_full_pt_social_sciences.yaml |   8 +
 .../full/pt/_global_mmlu_full_pt_stem.yaml    |   8 +
 .../global_mmlu/full/pt/_pt_template_yaml     |  16 ++
 .../global_mmlu_full_pt_abstract_algebra.yaml |   5 +
 .../full/pt/global_mmlu_full_pt_anatomy.yaml  |   5 +
 .../pt/global_mmlu_full_pt_astronomy.yaml     |   5 +
 .../global_mmlu_full_pt_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_pt_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_pt_college_biology.yaml  |   5 +
 ...global_mmlu_full_pt_college_chemistry.yaml |   5 +
 ...mmlu_full_pt_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_pt_college_mathematics.yaml |   5 +
 .../global_mmlu_full_pt_college_medicine.yaml |   5 +
 .../global_mmlu_full_pt_college_physics.yaml  |   5 +
 ...global_mmlu_full_pt_computer_security.yaml |   5 +
 ...lobal_mmlu_full_pt_conceptual_physics.yaml |   5 +
 .../pt/global_mmlu_full_pt_econometrics.yaml  |   5 +
 ...l_mmlu_full_pt_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_pt_elementary_mathematics.yaml |   5 +
 .../pt/global_mmlu_full_pt_formal_logic.yaml  |   5 +
 .../pt/global_mmlu_full_pt_global_facts.yaml  |   5 +
 ...obal_mmlu_full_pt_high_school_biology.yaml |   5 +
 ...al_mmlu_full_pt_high_school_chemistry.yaml |   5 +
 ..._full_pt_high_school_computer_science.yaml |   5 +
 ..._full_pt_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_pt_high_school_geography.yaml |   5 +
 ...t_high_school_government_and_politics.yaml |   5 +
 ...lu_full_pt_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_pt_high_school_mathematics.yaml |   5 +
 ...lu_full_pt_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_pt_high_school_physics.yaml |   5 +
 ...l_mmlu_full_pt_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_pt_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_pt_high_school_us_history.yaml |   5 +
 ...mlu_full_pt_high_school_world_history.yaml |   5 +
 .../pt/global_mmlu_full_pt_human_aging.yaml   |   5 +
 .../global_mmlu_full_pt_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_pt_international_law.yaml |   5 +
 .../pt/global_mmlu_full_pt_jurisprudence.yaml |   5 +
 ...global_mmlu_full_pt_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_pt_machine_learning.yaml |   5 +
 .../pt/global_mmlu_full_pt_management.yaml    |   5 +
 .../pt/global_mmlu_full_pt_marketing.yaml     |   5 +
 .../global_mmlu_full_pt_medical_genetics.yaml |   5 +
 .../pt/global_mmlu_full_pt_miscellaneous.yaml |   5 +
 .../global_mmlu_full_pt_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_pt_moral_scenarios.yaml  |   5 +
 .../pt/global_mmlu_full_pt_nutrition.yaml     |   5 +
 .../pt/global_mmlu_full_pt_philosophy.yaml    |   5 +
 .../pt/global_mmlu_full_pt_prehistory.yaml    |   5 +
 ..._mmlu_full_pt_professional_accounting.yaml |   5 +
 .../global_mmlu_full_pt_professional_law.yaml |   5 +
 ...al_mmlu_full_pt_professional_medicine.yaml |   5 +
 ..._mmlu_full_pt_professional_psychology.yaml |   5 +
 .../global_mmlu_full_pt_public_relations.yaml |   5 +
 .../global_mmlu_full_pt_security_studies.yaml |   5 +
 .../pt/global_mmlu_full_pt_sociology.yaml     |   5 +
 ...global_mmlu_full_pt_us_foreign_policy.yaml |   5 +
 .../full/pt/global_mmlu_full_pt_virology.yaml |   5 +
 .../global_mmlu_full_pt_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/pt/utils.py  |  73 +++++
 .../full/ro/_global_mmlu_full_ro.yaml         |  11 +
 .../ro/_global_mmlu_full_ro_humanities.yaml   |   8 +
 .../full/ro/_global_mmlu_full_ro_other.yaml   |   8 +
 .../_global_mmlu_full_ro_social_sciences.yaml |   8 +
 .../full/ro/_global_mmlu_full_ro_stem.yaml    |   8 +
 .../global_mmlu/full/ro/_ro_template_yaml     |  16 ++
 .../global_mmlu_full_ro_abstract_algebra.yaml |   5 +
 .../full/ro/global_mmlu_full_ro_anatomy.yaml  |   5 +
 .../ro/global_mmlu_full_ro_astronomy.yaml     |   5 +
 .../global_mmlu_full_ro_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_ro_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_ro_college_biology.yaml  |   5 +
 ...global_mmlu_full_ro_college_chemistry.yaml |   5 +
 ...mmlu_full_ro_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_ro_college_mathematics.yaml |   5 +
 .../global_mmlu_full_ro_college_medicine.yaml |   5 +
 .../global_mmlu_full_ro_college_physics.yaml  |   5 +
 ...global_mmlu_full_ro_computer_security.yaml |   5 +
 ...lobal_mmlu_full_ro_conceptual_physics.yaml |   5 +
 .../ro/global_mmlu_full_ro_econometrics.yaml  |   5 +
 ...l_mmlu_full_ro_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_ro_elementary_mathematics.yaml |   5 +
 .../ro/global_mmlu_full_ro_formal_logic.yaml  |   5 +
 .../ro/global_mmlu_full_ro_global_facts.yaml  |   5 +
 ...obal_mmlu_full_ro_high_school_biology.yaml |   5 +
 ...al_mmlu_full_ro_high_school_chemistry.yaml |   5 +
 ..._full_ro_high_school_computer_science.yaml |   5 +
 ..._full_ro_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_ro_high_school_geography.yaml |   5 +
 ...o_high_school_government_and_politics.yaml |   5 +
 ...lu_full_ro_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_ro_high_school_mathematics.yaml |   5 +
 ...lu_full_ro_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_ro_high_school_physics.yaml |   5 +
 ...l_mmlu_full_ro_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_ro_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_ro_high_school_us_history.yaml |   5 +
 ...mlu_full_ro_high_school_world_history.yaml |   5 +
 .../ro/global_mmlu_full_ro_human_aging.yaml   |   5 +
 .../global_mmlu_full_ro_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_ro_international_law.yaml |   5 +
 .../ro/global_mmlu_full_ro_jurisprudence.yaml |   5 +
 ...global_mmlu_full_ro_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_ro_machine_learning.yaml |   5 +
 .../ro/global_mmlu_full_ro_management.yaml    |   5 +
 .../ro/global_mmlu_full_ro_marketing.yaml     |   5 +
 .../global_mmlu_full_ro_medical_genetics.yaml |   5 +
 .../ro/global_mmlu_full_ro_miscellaneous.yaml |   5 +
 .../global_mmlu_full_ro_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_ro_moral_scenarios.yaml  |   5 +
 .../ro/global_mmlu_full_ro_nutrition.yaml     |   5 +
 .../ro/global_mmlu_full_ro_philosophy.yaml    |   5 +
 .../ro/global_mmlu_full_ro_prehistory.yaml    |   5 +
 ..._mmlu_full_ro_professional_accounting.yaml |   5 +
 .../global_mmlu_full_ro_professional_law.yaml |   5 +
 ...al_mmlu_full_ro_professional_medicine.yaml |   5 +
 ..._mmlu_full_ro_professional_psychology.yaml |   5 +
 .../global_mmlu_full_ro_public_relations.yaml |   5 +
 .../global_mmlu_full_ro_security_studies.yaml |   5 +
 .../ro/global_mmlu_full_ro_sociology.yaml     |   5 +
 ...global_mmlu_full_ro_us_foreign_policy.yaml |   5 +
 .../full/ro/global_mmlu_full_ro_virology.yaml |   5 +
 .../global_mmlu_full_ro_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/ro/utils.py  |  73 +++++
 .../full/ru/_global_mmlu_full_ru.yaml         |  11 +
 .../ru/_global_mmlu_full_ru_humanities.yaml   |   8 +
 .../full/ru/_global_mmlu_full_ru_other.yaml   |   8 +
 .../_global_mmlu_full_ru_social_sciences.yaml |   8 +
 .../full/ru/_global_mmlu_full_ru_stem.yaml    |   8 +
 .../global_mmlu/full/ru/_ru_template_yaml     |  16 ++
 .../global_mmlu_full_ru_abstract_algebra.yaml |   5 +
 .../full/ru/global_mmlu_full_ru_anatomy.yaml  |   5 +
 .../ru/global_mmlu_full_ru_astronomy.yaml     |   5 +
 .../global_mmlu_full_ru_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_ru_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_ru_college_biology.yaml  |   5 +
 ...global_mmlu_full_ru_college_chemistry.yaml |   5 +
 ...mmlu_full_ru_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_ru_college_mathematics.yaml |   5 +
 .../global_mmlu_full_ru_college_medicine.yaml |   5 +
 .../global_mmlu_full_ru_college_physics.yaml  |   5 +
 ...global_mmlu_full_ru_computer_security.yaml |   5 +
 ...lobal_mmlu_full_ru_conceptual_physics.yaml |   5 +
 .../ru/global_mmlu_full_ru_econometrics.yaml  |   5 +
 ...l_mmlu_full_ru_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_ru_elementary_mathematics.yaml |   5 +
 .../ru/global_mmlu_full_ru_formal_logic.yaml  |   5 +
 .../ru/global_mmlu_full_ru_global_facts.yaml  |   5 +
 ...obal_mmlu_full_ru_high_school_biology.yaml |   5 +
 ...al_mmlu_full_ru_high_school_chemistry.yaml |   5 +
 ..._full_ru_high_school_computer_science.yaml |   5 +
 ..._full_ru_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_ru_high_school_geography.yaml |   5 +
 ...u_high_school_government_and_politics.yaml |   5 +
 ...lu_full_ru_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_ru_high_school_mathematics.yaml |   5 +
 ...lu_full_ru_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_ru_high_school_physics.yaml |   5 +
 ...l_mmlu_full_ru_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_ru_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_ru_high_school_us_history.yaml |   5 +
 ...mlu_full_ru_high_school_world_history.yaml |   5 +
 .../ru/global_mmlu_full_ru_human_aging.yaml   |   5 +
 .../global_mmlu_full_ru_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_ru_international_law.yaml |   5 +
 .../ru/global_mmlu_full_ru_jurisprudence.yaml |   5 +
 ...global_mmlu_full_ru_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_ru_machine_learning.yaml |   5 +
 .../ru/global_mmlu_full_ru_management.yaml    |   5 +
 .../ru/global_mmlu_full_ru_marketing.yaml     |   5 +
 .../global_mmlu_full_ru_medical_genetics.yaml |   5 +
 .../ru/global_mmlu_full_ru_miscellaneous.yaml |   5 +
 .../global_mmlu_full_ru_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_ru_moral_scenarios.yaml  |   5 +
 .../ru/global_mmlu_full_ru_nutrition.yaml     |   5 +
 .../ru/global_mmlu_full_ru_philosophy.yaml    |   5 +
 .../ru/global_mmlu_full_ru_prehistory.yaml    |   5 +
 ..._mmlu_full_ru_professional_accounting.yaml |   5 +
 .../global_mmlu_full_ru_professional_law.yaml |   5 +
 ...al_mmlu_full_ru_professional_medicine.yaml |   5 +
 ..._mmlu_full_ru_professional_psychology.yaml |   5 +
 .../global_mmlu_full_ru_public_relations.yaml |   5 +
 .../global_mmlu_full_ru_security_studies.yaml |   5 +
 .../ru/global_mmlu_full_ru_sociology.yaml     |   5 +
 ...global_mmlu_full_ru_us_foreign_policy.yaml |   5 +
 .../full/ru/global_mmlu_full_ru_virology.yaml |   5 +
 .../global_mmlu_full_ru_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/ru/utils.py  |  73 +++++
 .../full/si/_global_mmlu_full_si.yaml         |  11 +
 .../si/_global_mmlu_full_si_humanities.yaml   |   8 +
 .../full/si/_global_mmlu_full_si_other.yaml   |   8 +
 .../_global_mmlu_full_si_social_sciences.yaml |   8 +
 .../full/si/_global_mmlu_full_si_stem.yaml    |   8 +
 .../global_mmlu/full/si/_si_template_yaml     |  16 ++
 .../global_mmlu_full_si_abstract_algebra.yaml |   5 +
 .../full/si/global_mmlu_full_si_anatomy.yaml  |   5 +
 .../si/global_mmlu_full_si_astronomy.yaml     |   5 +
 .../global_mmlu_full_si_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_si_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_si_college_biology.yaml  |   5 +
 ...global_mmlu_full_si_college_chemistry.yaml |   5 +
 ...mmlu_full_si_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_si_college_mathematics.yaml |   5 +
 .../global_mmlu_full_si_college_medicine.yaml |   5 +
 .../global_mmlu_full_si_college_physics.yaml  |   5 +
 ...global_mmlu_full_si_computer_security.yaml |   5 +
 ...lobal_mmlu_full_si_conceptual_physics.yaml |   5 +
 .../si/global_mmlu_full_si_econometrics.yaml  |   5 +
 ...l_mmlu_full_si_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_si_elementary_mathematics.yaml |   5 +
 .../si/global_mmlu_full_si_formal_logic.yaml  |   5 +
 .../si/global_mmlu_full_si_global_facts.yaml  |   5 +
 ...obal_mmlu_full_si_high_school_biology.yaml |   5 +
 ...al_mmlu_full_si_high_school_chemistry.yaml |   5 +
 ..._full_si_high_school_computer_science.yaml |   5 +
 ..._full_si_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_si_high_school_geography.yaml |   5 +
 ...i_high_school_government_and_politics.yaml |   5 +
 ...lu_full_si_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_si_high_school_mathematics.yaml |   5 +
 ...lu_full_si_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_si_high_school_physics.yaml |   5 +
 ...l_mmlu_full_si_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_si_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_si_high_school_us_history.yaml |   5 +
 ...mlu_full_si_high_school_world_history.yaml |   5 +
 .../si/global_mmlu_full_si_human_aging.yaml   |   5 +
 .../global_mmlu_full_si_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_si_international_law.yaml |   5 +
 .../si/global_mmlu_full_si_jurisprudence.yaml |   5 +
 ...global_mmlu_full_si_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_si_machine_learning.yaml |   5 +
 .../si/global_mmlu_full_si_management.yaml    |   5 +
 .../si/global_mmlu_full_si_marketing.yaml     |   5 +
 .../global_mmlu_full_si_medical_genetics.yaml |   5 +
 .../si/global_mmlu_full_si_miscellaneous.yaml |   5 +
 .../global_mmlu_full_si_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_si_moral_scenarios.yaml  |   5 +
 .../si/global_mmlu_full_si_nutrition.yaml     |   5 +
 .../si/global_mmlu_full_si_philosophy.yaml    |   5 +
 .../si/global_mmlu_full_si_prehistory.yaml    |   5 +
 ..._mmlu_full_si_professional_accounting.yaml |   5 +
 .../global_mmlu_full_si_professional_law.yaml |   5 +
 ...al_mmlu_full_si_professional_medicine.yaml |   5 +
 ..._mmlu_full_si_professional_psychology.yaml |   5 +
 .../global_mmlu_full_si_public_relations.yaml |   5 +
 .../global_mmlu_full_si_security_studies.yaml |   5 +
 .../si/global_mmlu_full_si_sociology.yaml     |   5 +
 ...global_mmlu_full_si_us_foreign_policy.yaml |   5 +
 .../full/si/global_mmlu_full_si_virology.yaml |   5 +
 .../global_mmlu_full_si_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/si/utils.py  |  73 +++++
 .../full/sn/_global_mmlu_full_sn.yaml         |  11 +
 .../sn/_global_mmlu_full_sn_humanities.yaml   |   8 +
 .../full/sn/_global_mmlu_full_sn_other.yaml   |   8 +
 .../_global_mmlu_full_sn_social_sciences.yaml |   8 +
 .../full/sn/_global_mmlu_full_sn_stem.yaml    |   8 +
 .../global_mmlu/full/sn/_sn_template_yaml     |  16 ++
 .../global_mmlu_full_sn_abstract_algebra.yaml |   5 +
 .../full/sn/global_mmlu_full_sn_anatomy.yaml  |   5 +
 .../sn/global_mmlu_full_sn_astronomy.yaml     |   5 +
 .../global_mmlu_full_sn_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_sn_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_sn_college_biology.yaml  |   5 +
 ...global_mmlu_full_sn_college_chemistry.yaml |   5 +
 ...mmlu_full_sn_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_sn_college_mathematics.yaml |   5 +
 .../global_mmlu_full_sn_college_medicine.yaml |   5 +
 .../global_mmlu_full_sn_college_physics.yaml  |   5 +
 ...global_mmlu_full_sn_computer_security.yaml |   5 +
 ...lobal_mmlu_full_sn_conceptual_physics.yaml |   5 +
 .../sn/global_mmlu_full_sn_econometrics.yaml  |   5 +
 ...l_mmlu_full_sn_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_sn_elementary_mathematics.yaml |   5 +
 .../sn/global_mmlu_full_sn_formal_logic.yaml  |   5 +
 .../sn/global_mmlu_full_sn_global_facts.yaml  |   5 +
 ...obal_mmlu_full_sn_high_school_biology.yaml |   5 +
 ...al_mmlu_full_sn_high_school_chemistry.yaml |   5 +
 ..._full_sn_high_school_computer_science.yaml |   5 +
 ..._full_sn_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_sn_high_school_geography.yaml |   5 +
 ...n_high_school_government_and_politics.yaml |   5 +
 ...lu_full_sn_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_sn_high_school_mathematics.yaml |   5 +
 ...lu_full_sn_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_sn_high_school_physics.yaml |   5 +
 ...l_mmlu_full_sn_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_sn_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_sn_high_school_us_history.yaml |   5 +
 ...mlu_full_sn_high_school_world_history.yaml |   5 +
 .../sn/global_mmlu_full_sn_human_aging.yaml   |   5 +
 .../global_mmlu_full_sn_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_sn_international_law.yaml |   5 +
 .../sn/global_mmlu_full_sn_jurisprudence.yaml |   5 +
 ...global_mmlu_full_sn_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_sn_machine_learning.yaml |   5 +
 .../sn/global_mmlu_full_sn_management.yaml    |   5 +
 .../sn/global_mmlu_full_sn_marketing.yaml     |   5 +
 .../global_mmlu_full_sn_medical_genetics.yaml |   5 +
 .../sn/global_mmlu_full_sn_miscellaneous.yaml |   5 +
 .../global_mmlu_full_sn_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_sn_moral_scenarios.yaml  |   5 +
 .../sn/global_mmlu_full_sn_nutrition.yaml     |   5 +
 .../sn/global_mmlu_full_sn_philosophy.yaml    |   5 +
 .../sn/global_mmlu_full_sn_prehistory.yaml    |   5 +
 ..._mmlu_full_sn_professional_accounting.yaml |   5 +
 .../global_mmlu_full_sn_professional_law.yaml |   5 +
 ...al_mmlu_full_sn_professional_medicine.yaml |   5 +
 ..._mmlu_full_sn_professional_psychology.yaml |   5 +
 .../global_mmlu_full_sn_public_relations.yaml |   5 +
 .../global_mmlu_full_sn_security_studies.yaml |   5 +
 .../sn/global_mmlu_full_sn_sociology.yaml     |   5 +
 ...global_mmlu_full_sn_us_foreign_policy.yaml |   5 +
 .../full/sn/global_mmlu_full_sn_virology.yaml |   5 +
 .../global_mmlu_full_sn_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/sn/utils.py  |  73 +++++
 .../full/so/_global_mmlu_full_so.yaml         |  11 +
 .../so/_global_mmlu_full_so_humanities.yaml   |   8 +
 .../full/so/_global_mmlu_full_so_other.yaml   |   8 +
 .../_global_mmlu_full_so_social_sciences.yaml |   8 +
 .../full/so/_global_mmlu_full_so_stem.yaml    |   8 +
 .../global_mmlu/full/so/_so_template_yaml     |  16 ++
 .../global_mmlu_full_so_abstract_algebra.yaml |   5 +
 .../full/so/global_mmlu_full_so_anatomy.yaml  |   5 +
 .../so/global_mmlu_full_so_astronomy.yaml     |   5 +
 .../global_mmlu_full_so_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_so_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_so_college_biology.yaml  |   5 +
 ...global_mmlu_full_so_college_chemistry.yaml |   5 +
 ...mmlu_full_so_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_so_college_mathematics.yaml |   5 +
 .../global_mmlu_full_so_college_medicine.yaml |   5 +
 .../global_mmlu_full_so_college_physics.yaml  |   5 +
 ...global_mmlu_full_so_computer_security.yaml |   5 +
 ...lobal_mmlu_full_so_conceptual_physics.yaml |   5 +
 .../so/global_mmlu_full_so_econometrics.yaml  |   5 +
 ...l_mmlu_full_so_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_so_elementary_mathematics.yaml |   5 +
 .../so/global_mmlu_full_so_formal_logic.yaml  |   5 +
 .../so/global_mmlu_full_so_global_facts.yaml  |   5 +
 ...obal_mmlu_full_so_high_school_biology.yaml |   5 +
 ...al_mmlu_full_so_high_school_chemistry.yaml |   5 +
 ..._full_so_high_school_computer_science.yaml |   5 +
 ..._full_so_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_so_high_school_geography.yaml |   5 +
 ...o_high_school_government_and_politics.yaml |   5 +
 ...lu_full_so_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_so_high_school_mathematics.yaml |   5 +
 ...lu_full_so_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_so_high_school_physics.yaml |   5 +
 ...l_mmlu_full_so_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_so_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_so_high_school_us_history.yaml |   5 +
 ...mlu_full_so_high_school_world_history.yaml |   5 +
 .../so/global_mmlu_full_so_human_aging.yaml   |   5 +
 .../global_mmlu_full_so_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_so_international_law.yaml |   5 +
 .../so/global_mmlu_full_so_jurisprudence.yaml |   5 +
 ...global_mmlu_full_so_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_so_machine_learning.yaml |   5 +
 .../so/global_mmlu_full_so_management.yaml    |   5 +
 .../so/global_mmlu_full_so_marketing.yaml     |   5 +
 .../global_mmlu_full_so_medical_genetics.yaml |   5 +
 .../so/global_mmlu_full_so_miscellaneous.yaml |   5 +
 .../global_mmlu_full_so_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_so_moral_scenarios.yaml  |   5 +
 .../so/global_mmlu_full_so_nutrition.yaml     |   5 +
 .../so/global_mmlu_full_so_philosophy.yaml    |   5 +
 .../so/global_mmlu_full_so_prehistory.yaml    |   5 +
 ..._mmlu_full_so_professional_accounting.yaml |   5 +
 .../global_mmlu_full_so_professional_law.yaml |   5 +
 ...al_mmlu_full_so_professional_medicine.yaml |   5 +
 ..._mmlu_full_so_professional_psychology.yaml |   5 +
 .../global_mmlu_full_so_public_relations.yaml |   5 +
 .../global_mmlu_full_so_security_studies.yaml |   5 +
 .../so/global_mmlu_full_so_sociology.yaml     |   5 +
 ...global_mmlu_full_so_us_foreign_policy.yaml |   5 +
 .../full/so/global_mmlu_full_so_virology.yaml |   5 +
 .../global_mmlu_full_so_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/so/utils.py  |  73 +++++
 .../full/sr/_global_mmlu_full_sr.yaml         |  11 +
 .../sr/_global_mmlu_full_sr_humanities.yaml   |   8 +
 .../full/sr/_global_mmlu_full_sr_other.yaml   |   8 +
 .../_global_mmlu_full_sr_social_sciences.yaml |   8 +
 .../full/sr/_global_mmlu_full_sr_stem.yaml    |   8 +
 .../global_mmlu/full/sr/_sr_template_yaml     |  16 ++
 .../global_mmlu_full_sr_abstract_algebra.yaml |   5 +
 .../full/sr/global_mmlu_full_sr_anatomy.yaml  |   5 +
 .../sr/global_mmlu_full_sr_astronomy.yaml     |   5 +
 .../global_mmlu_full_sr_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_sr_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_sr_college_biology.yaml  |   5 +
 ...global_mmlu_full_sr_college_chemistry.yaml |   5 +
 ...mmlu_full_sr_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_sr_college_mathematics.yaml |   5 +
 .../global_mmlu_full_sr_college_medicine.yaml |   5 +
 .../global_mmlu_full_sr_college_physics.yaml  |   5 +
 ...global_mmlu_full_sr_computer_security.yaml |   5 +
 ...lobal_mmlu_full_sr_conceptual_physics.yaml |   5 +
 .../sr/global_mmlu_full_sr_econometrics.yaml  |   5 +
 ...l_mmlu_full_sr_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_sr_elementary_mathematics.yaml |   5 +
 .../sr/global_mmlu_full_sr_formal_logic.yaml  |   5 +
 .../sr/global_mmlu_full_sr_global_facts.yaml  |   5 +
 ...obal_mmlu_full_sr_high_school_biology.yaml |   5 +
 ...al_mmlu_full_sr_high_school_chemistry.yaml |   5 +
 ..._full_sr_high_school_computer_science.yaml |   5 +
 ..._full_sr_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_sr_high_school_geography.yaml |   5 +
 ...r_high_school_government_and_politics.yaml |   5 +
 ...lu_full_sr_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_sr_high_school_mathematics.yaml |   5 +
 ...lu_full_sr_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_sr_high_school_physics.yaml |   5 +
 ...l_mmlu_full_sr_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_sr_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_sr_high_school_us_history.yaml |   5 +
 ...mlu_full_sr_high_school_world_history.yaml |   5 +
 .../sr/global_mmlu_full_sr_human_aging.yaml   |   5 +
 .../global_mmlu_full_sr_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_sr_international_law.yaml |   5 +
 .../sr/global_mmlu_full_sr_jurisprudence.yaml |   5 +
 ...global_mmlu_full_sr_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_sr_machine_learning.yaml |   5 +
 .../sr/global_mmlu_full_sr_management.yaml    |   5 +
 .../sr/global_mmlu_full_sr_marketing.yaml     |   5 +
 .../global_mmlu_full_sr_medical_genetics.yaml |   5 +
 .../sr/global_mmlu_full_sr_miscellaneous.yaml |   5 +
 .../global_mmlu_full_sr_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_sr_moral_scenarios.yaml  |   5 +
 .../sr/global_mmlu_full_sr_nutrition.yaml     |   5 +
 .../sr/global_mmlu_full_sr_philosophy.yaml    |   5 +
 .../sr/global_mmlu_full_sr_prehistory.yaml    |   5 +
 ..._mmlu_full_sr_professional_accounting.yaml |   5 +
 .../global_mmlu_full_sr_professional_law.yaml |   5 +
 ...al_mmlu_full_sr_professional_medicine.yaml |   5 +
 ..._mmlu_full_sr_professional_psychology.yaml |   5 +
 .../global_mmlu_full_sr_public_relations.yaml |   5 +
 .../global_mmlu_full_sr_security_studies.yaml |   5 +
 .../sr/global_mmlu_full_sr_sociology.yaml     |   5 +
 ...global_mmlu_full_sr_us_foreign_policy.yaml |   5 +
 .../full/sr/global_mmlu_full_sr_virology.yaml |   5 +
 .../global_mmlu_full_sr_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/sr/utils.py  |  73 +++++
 .../full/sv/_global_mmlu_full_sv.yaml         |  11 +
 .../sv/_global_mmlu_full_sv_humanities.yaml   |   8 +
 .../full/sv/_global_mmlu_full_sv_other.yaml   |   8 +
 .../_global_mmlu_full_sv_social_sciences.yaml |   8 +
 .../full/sv/_global_mmlu_full_sv_stem.yaml    |   8 +
 .../global_mmlu/full/sv/_sv_template_yaml     |  16 ++
 .../global_mmlu_full_sv_abstract_algebra.yaml |   5 +
 .../full/sv/global_mmlu_full_sv_anatomy.yaml  |   5 +
 .../sv/global_mmlu_full_sv_astronomy.yaml     |   5 +
 .../global_mmlu_full_sv_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_sv_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_sv_college_biology.yaml  |   5 +
 ...global_mmlu_full_sv_college_chemistry.yaml |   5 +
 ...mmlu_full_sv_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_sv_college_mathematics.yaml |   5 +
 .../global_mmlu_full_sv_college_medicine.yaml |   5 +
 .../global_mmlu_full_sv_college_physics.yaml  |   5 +
 ...global_mmlu_full_sv_computer_security.yaml |   5 +
 ...lobal_mmlu_full_sv_conceptual_physics.yaml |   5 +
 .../sv/global_mmlu_full_sv_econometrics.yaml  |   5 +
 ...l_mmlu_full_sv_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_sv_elementary_mathematics.yaml |   5 +
 .../sv/global_mmlu_full_sv_formal_logic.yaml  |   5 +
 .../sv/global_mmlu_full_sv_global_facts.yaml  |   5 +
 ...obal_mmlu_full_sv_high_school_biology.yaml |   5 +
 ...al_mmlu_full_sv_high_school_chemistry.yaml |   5 +
 ..._full_sv_high_school_computer_science.yaml |   5 +
 ..._full_sv_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_sv_high_school_geography.yaml |   5 +
 ...v_high_school_government_and_politics.yaml |   5 +
 ...lu_full_sv_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_sv_high_school_mathematics.yaml |   5 +
 ...lu_full_sv_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_sv_high_school_physics.yaml |   5 +
 ...l_mmlu_full_sv_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_sv_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_sv_high_school_us_history.yaml |   5 +
 ...mlu_full_sv_high_school_world_history.yaml |   5 +
 .../sv/global_mmlu_full_sv_human_aging.yaml   |   5 +
 .../global_mmlu_full_sv_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_sv_international_law.yaml |   5 +
 .../sv/global_mmlu_full_sv_jurisprudence.yaml |   5 +
 ...global_mmlu_full_sv_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_sv_machine_learning.yaml |   5 +
 .../sv/global_mmlu_full_sv_management.yaml    |   5 +
 .../sv/global_mmlu_full_sv_marketing.yaml     |   5 +
 .../global_mmlu_full_sv_medical_genetics.yaml |   5 +
 .../sv/global_mmlu_full_sv_miscellaneous.yaml |   5 +
 .../global_mmlu_full_sv_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_sv_moral_scenarios.yaml  |   5 +
 .../sv/global_mmlu_full_sv_nutrition.yaml     |   5 +
 .../sv/global_mmlu_full_sv_philosophy.yaml    |   5 +
 .../sv/global_mmlu_full_sv_prehistory.yaml    |   5 +
 ..._mmlu_full_sv_professional_accounting.yaml |   5 +
 .../global_mmlu_full_sv_professional_law.yaml |   5 +
 ...al_mmlu_full_sv_professional_medicine.yaml |   5 +
 ..._mmlu_full_sv_professional_psychology.yaml |   5 +
 .../global_mmlu_full_sv_public_relations.yaml |   5 +
 .../global_mmlu_full_sv_security_studies.yaml |   5 +
 .../sv/global_mmlu_full_sv_sociology.yaml     |   5 +
 ...global_mmlu_full_sv_us_foreign_policy.yaml |   5 +
 .../full/sv/global_mmlu_full_sv_virology.yaml |   5 +
 .../global_mmlu_full_sv_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/sv/utils.py  |  73 +++++
 .../full/sw/_global_mmlu_full_sw.yaml         |  11 +
 .../sw/_global_mmlu_full_sw_humanities.yaml   |   8 +
 .../full/sw/_global_mmlu_full_sw_other.yaml   |   8 +
 .../_global_mmlu_full_sw_social_sciences.yaml |   8 +
 .../full/sw/_global_mmlu_full_sw_stem.yaml    |   8 +
 .../global_mmlu/full/sw/_sw_template_yaml     |  16 ++
 .../global_mmlu_full_sw_abstract_algebra.yaml |   5 +
 .../full/sw/global_mmlu_full_sw_anatomy.yaml  |   5 +
 .../sw/global_mmlu_full_sw_astronomy.yaml     |   5 +
 .../global_mmlu_full_sw_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_sw_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_sw_college_biology.yaml  |   5 +
 ...global_mmlu_full_sw_college_chemistry.yaml |   5 +
 ...mmlu_full_sw_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_sw_college_mathematics.yaml |   5 +
 .../global_mmlu_full_sw_college_medicine.yaml |   5 +
 .../global_mmlu_full_sw_college_physics.yaml  |   5 +
 ...global_mmlu_full_sw_computer_security.yaml |   5 +
 ...lobal_mmlu_full_sw_conceptual_physics.yaml |   5 +
 .../sw/global_mmlu_full_sw_econometrics.yaml  |   5 +
 ...l_mmlu_full_sw_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_sw_elementary_mathematics.yaml |   5 +
 .../sw/global_mmlu_full_sw_formal_logic.yaml  |   5 +
 .../sw/global_mmlu_full_sw_global_facts.yaml  |   5 +
 ...obal_mmlu_full_sw_high_school_biology.yaml |   5 +
 ...al_mmlu_full_sw_high_school_chemistry.yaml |   5 +
 ..._full_sw_high_school_computer_science.yaml |   5 +
 ..._full_sw_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_sw_high_school_geography.yaml |   5 +
 ...w_high_school_government_and_politics.yaml |   5 +
 ...lu_full_sw_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_sw_high_school_mathematics.yaml |   5 +
 ...lu_full_sw_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_sw_high_school_physics.yaml |   5 +
 ...l_mmlu_full_sw_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_sw_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_sw_high_school_us_history.yaml |   5 +
 ...mlu_full_sw_high_school_world_history.yaml |   5 +
 .../sw/global_mmlu_full_sw_human_aging.yaml   |   5 +
 .../global_mmlu_full_sw_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_sw_international_law.yaml |   5 +
 .../sw/global_mmlu_full_sw_jurisprudence.yaml |   5 +
 ...global_mmlu_full_sw_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_sw_machine_learning.yaml |   5 +
 .../sw/global_mmlu_full_sw_management.yaml    |   5 +
 .../sw/global_mmlu_full_sw_marketing.yaml     |   5 +
 .../global_mmlu_full_sw_medical_genetics.yaml |   5 +
 .../sw/global_mmlu_full_sw_miscellaneous.yaml |   5 +
 .../global_mmlu_full_sw_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_sw_moral_scenarios.yaml  |   5 +
 .../sw/global_mmlu_full_sw_nutrition.yaml     |   5 +
 .../sw/global_mmlu_full_sw_philosophy.yaml    |   5 +
 .../sw/global_mmlu_full_sw_prehistory.yaml    |   5 +
 ..._mmlu_full_sw_professional_accounting.yaml |   5 +
 .../global_mmlu_full_sw_professional_law.yaml |   5 +
 ...al_mmlu_full_sw_professional_medicine.yaml |   5 +
 ..._mmlu_full_sw_professional_psychology.yaml |   5 +
 .../global_mmlu_full_sw_public_relations.yaml |   5 +
 .../global_mmlu_full_sw_security_studies.yaml |   5 +
 .../sw/global_mmlu_full_sw_sociology.yaml     |   5 +
 ...global_mmlu_full_sw_us_foreign_policy.yaml |   5 +
 .../full/sw/global_mmlu_full_sw_virology.yaml |   5 +
 .../global_mmlu_full_sw_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/sw/utils.py  |  73 +++++
 .../full/te/_global_mmlu_full_te.yaml         |  11 +
 .../te/_global_mmlu_full_te_humanities.yaml   |   8 +
 .../full/te/_global_mmlu_full_te_other.yaml   |   8 +
 .../_global_mmlu_full_te_social_sciences.yaml |   8 +
 .../full/te/_global_mmlu_full_te_stem.yaml    |   8 +
 .../global_mmlu/full/te/_te_template_yaml     |  16 ++
 .../global_mmlu_full_te_abstract_algebra.yaml |   5 +
 .../full/te/global_mmlu_full_te_anatomy.yaml  |   5 +
 .../te/global_mmlu_full_te_astronomy.yaml     |   5 +
 .../global_mmlu_full_te_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_te_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_te_college_biology.yaml  |   5 +
 ...global_mmlu_full_te_college_chemistry.yaml |   5 +
 ...mmlu_full_te_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_te_college_mathematics.yaml |   5 +
 .../global_mmlu_full_te_college_medicine.yaml |   5 +
 .../global_mmlu_full_te_college_physics.yaml  |   5 +
 ...global_mmlu_full_te_computer_security.yaml |   5 +
 ...lobal_mmlu_full_te_conceptual_physics.yaml |   5 +
 .../te/global_mmlu_full_te_econometrics.yaml  |   5 +
 ...l_mmlu_full_te_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_te_elementary_mathematics.yaml |   5 +
 .../te/global_mmlu_full_te_formal_logic.yaml  |   5 +
 .../te/global_mmlu_full_te_global_facts.yaml  |   5 +
 ...obal_mmlu_full_te_high_school_biology.yaml |   5 +
 ...al_mmlu_full_te_high_school_chemistry.yaml |   5 +
 ..._full_te_high_school_computer_science.yaml |   5 +
 ..._full_te_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_te_high_school_geography.yaml |   5 +
 ...e_high_school_government_and_politics.yaml |   5 +
 ...lu_full_te_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_te_high_school_mathematics.yaml |   5 +
 ...lu_full_te_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_te_high_school_physics.yaml |   5 +
 ...l_mmlu_full_te_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_te_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_te_high_school_us_history.yaml |   5 +
 ...mlu_full_te_high_school_world_history.yaml |   5 +
 .../te/global_mmlu_full_te_human_aging.yaml   |   5 +
 .../global_mmlu_full_te_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_te_international_law.yaml |   5 +
 .../te/global_mmlu_full_te_jurisprudence.yaml |   5 +
 ...global_mmlu_full_te_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_te_machine_learning.yaml |   5 +
 .../te/global_mmlu_full_te_management.yaml    |   5 +
 .../te/global_mmlu_full_te_marketing.yaml     |   5 +
 .../global_mmlu_full_te_medical_genetics.yaml |   5 +
 .../te/global_mmlu_full_te_miscellaneous.yaml |   5 +
 .../global_mmlu_full_te_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_te_moral_scenarios.yaml  |   5 +
 .../te/global_mmlu_full_te_nutrition.yaml     |   5 +
 .../te/global_mmlu_full_te_philosophy.yaml    |   5 +
 .../te/global_mmlu_full_te_prehistory.yaml    |   5 +
 ..._mmlu_full_te_professional_accounting.yaml |   5 +
 .../global_mmlu_full_te_professional_law.yaml |   5 +
 ...al_mmlu_full_te_professional_medicine.yaml |   5 +
 ..._mmlu_full_te_professional_psychology.yaml |   5 +
 .../global_mmlu_full_te_public_relations.yaml |   5 +
 .../global_mmlu_full_te_security_studies.yaml |   5 +
 .../te/global_mmlu_full_te_sociology.yaml     |   5 +
 ...global_mmlu_full_te_us_foreign_policy.yaml |   5 +
 .../full/te/global_mmlu_full_te_virology.yaml |   5 +
 .../global_mmlu_full_te_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/te/utils.py  |  73 +++++
 .../full/tr/_global_mmlu_full_tr.yaml         |  11 +
 .../tr/_global_mmlu_full_tr_humanities.yaml   |   8 +
 .../full/tr/_global_mmlu_full_tr_other.yaml   |   8 +
 .../_global_mmlu_full_tr_social_sciences.yaml |   8 +
 .../full/tr/_global_mmlu_full_tr_stem.yaml    |   8 +
 .../global_mmlu/full/tr/_tr_template_yaml     |  16 ++
 .../global_mmlu_full_tr_abstract_algebra.yaml |   5 +
 .../full/tr/global_mmlu_full_tr_anatomy.yaml  |   5 +
 .../tr/global_mmlu_full_tr_astronomy.yaml     |   5 +
 .../global_mmlu_full_tr_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_tr_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_tr_college_biology.yaml  |   5 +
 ...global_mmlu_full_tr_college_chemistry.yaml |   5 +
 ...mmlu_full_tr_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_tr_college_mathematics.yaml |   5 +
 .../global_mmlu_full_tr_college_medicine.yaml |   5 +
 .../global_mmlu_full_tr_college_physics.yaml  |   5 +
 ...global_mmlu_full_tr_computer_security.yaml |   5 +
 ...lobal_mmlu_full_tr_conceptual_physics.yaml |   5 +
 .../tr/global_mmlu_full_tr_econometrics.yaml  |   5 +
 ...l_mmlu_full_tr_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_tr_elementary_mathematics.yaml |   5 +
 .../tr/global_mmlu_full_tr_formal_logic.yaml  |   5 +
 .../tr/global_mmlu_full_tr_global_facts.yaml  |   5 +
 ...obal_mmlu_full_tr_high_school_biology.yaml |   5 +
 ...al_mmlu_full_tr_high_school_chemistry.yaml |   5 +
 ..._full_tr_high_school_computer_science.yaml |   5 +
 ..._full_tr_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_tr_high_school_geography.yaml |   5 +
 ...r_high_school_government_and_politics.yaml |   5 +
 ...lu_full_tr_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_tr_high_school_mathematics.yaml |   5 +
 ...lu_full_tr_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_tr_high_school_physics.yaml |   5 +
 ...l_mmlu_full_tr_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_tr_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_tr_high_school_us_history.yaml |   5 +
 ...mlu_full_tr_high_school_world_history.yaml |   5 +
 .../tr/global_mmlu_full_tr_human_aging.yaml   |   5 +
 .../global_mmlu_full_tr_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_tr_international_law.yaml |   5 +
 .../tr/global_mmlu_full_tr_jurisprudence.yaml |   5 +
 ...global_mmlu_full_tr_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_tr_machine_learning.yaml |   5 +
 .../tr/global_mmlu_full_tr_management.yaml    |   5 +
 .../tr/global_mmlu_full_tr_marketing.yaml     |   5 +
 .../global_mmlu_full_tr_medical_genetics.yaml |   5 +
 .../tr/global_mmlu_full_tr_miscellaneous.yaml |   5 +
 .../global_mmlu_full_tr_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_tr_moral_scenarios.yaml  |   5 +
 .../tr/global_mmlu_full_tr_nutrition.yaml     |   5 +
 .../tr/global_mmlu_full_tr_philosophy.yaml    |   5 +
 .../tr/global_mmlu_full_tr_prehistory.yaml    |   5 +
 ..._mmlu_full_tr_professional_accounting.yaml |   5 +
 .../global_mmlu_full_tr_professional_law.yaml |   5 +
 ...al_mmlu_full_tr_professional_medicine.yaml |   5 +
 ..._mmlu_full_tr_professional_psychology.yaml |   5 +
 .../global_mmlu_full_tr_public_relations.yaml |   5 +
 .../global_mmlu_full_tr_security_studies.yaml |   5 +
 .../tr/global_mmlu_full_tr_sociology.yaml     |   5 +
 ...global_mmlu_full_tr_us_foreign_policy.yaml |   5 +
 .../full/tr/global_mmlu_full_tr_virology.yaml |   5 +
 .../global_mmlu_full_tr_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/tr/utils.py  |  73 +++++
 .../full/uk/_global_mmlu_full_uk.yaml         |  11 +
 .../uk/_global_mmlu_full_uk_humanities.yaml   |   8 +
 .../full/uk/_global_mmlu_full_uk_other.yaml   |   8 +
 .../_global_mmlu_full_uk_social_sciences.yaml |   8 +
 .../full/uk/_global_mmlu_full_uk_stem.yaml    |   8 +
 .../global_mmlu/full/uk/_uk_template_yaml     |  16 ++
 .../global_mmlu_full_uk_abstract_algebra.yaml |   5 +
 .../full/uk/global_mmlu_full_uk_anatomy.yaml  |   5 +
 .../uk/global_mmlu_full_uk_astronomy.yaml     |   5 +
 .../global_mmlu_full_uk_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_uk_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_uk_college_biology.yaml  |   5 +
 ...global_mmlu_full_uk_college_chemistry.yaml |   5 +
 ...mmlu_full_uk_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_uk_college_mathematics.yaml |   5 +
 .../global_mmlu_full_uk_college_medicine.yaml |   5 +
 .../global_mmlu_full_uk_college_physics.yaml  |   5 +
 ...global_mmlu_full_uk_computer_security.yaml |   5 +
 ...lobal_mmlu_full_uk_conceptual_physics.yaml |   5 +
 .../uk/global_mmlu_full_uk_econometrics.yaml  |   5 +
 ...l_mmlu_full_uk_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_uk_elementary_mathematics.yaml |   5 +
 .../uk/global_mmlu_full_uk_formal_logic.yaml  |   5 +
 .../uk/global_mmlu_full_uk_global_facts.yaml  |   5 +
 ...obal_mmlu_full_uk_high_school_biology.yaml |   5 +
 ...al_mmlu_full_uk_high_school_chemistry.yaml |   5 +
 ..._full_uk_high_school_computer_science.yaml |   5 +
 ..._full_uk_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_uk_high_school_geography.yaml |   5 +
 ...k_high_school_government_and_politics.yaml |   5 +
 ...lu_full_uk_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_uk_high_school_mathematics.yaml |   5 +
 ...lu_full_uk_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_uk_high_school_physics.yaml |   5 +
 ...l_mmlu_full_uk_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_uk_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_uk_high_school_us_history.yaml |   5 +
 ...mlu_full_uk_high_school_world_history.yaml |   5 +
 .../uk/global_mmlu_full_uk_human_aging.yaml   |   5 +
 .../global_mmlu_full_uk_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_uk_international_law.yaml |   5 +
 .../uk/global_mmlu_full_uk_jurisprudence.yaml |   5 +
 ...global_mmlu_full_uk_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_uk_machine_learning.yaml |   5 +
 .../uk/global_mmlu_full_uk_management.yaml    |   5 +
 .../uk/global_mmlu_full_uk_marketing.yaml     |   5 +
 .../global_mmlu_full_uk_medical_genetics.yaml |   5 +
 .../uk/global_mmlu_full_uk_miscellaneous.yaml |   5 +
 .../global_mmlu_full_uk_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_uk_moral_scenarios.yaml  |   5 +
 .../uk/global_mmlu_full_uk_nutrition.yaml     |   5 +
 .../uk/global_mmlu_full_uk_philosophy.yaml    |   5 +
 .../uk/global_mmlu_full_uk_prehistory.yaml    |   5 +
 ..._mmlu_full_uk_professional_accounting.yaml |   5 +
 .../global_mmlu_full_uk_professional_law.yaml |   5 +
 ...al_mmlu_full_uk_professional_medicine.yaml |   5 +
 ..._mmlu_full_uk_professional_psychology.yaml |   5 +
 .../global_mmlu_full_uk_public_relations.yaml |   5 +
 .../global_mmlu_full_uk_security_studies.yaml |   5 +
 .../uk/global_mmlu_full_uk_sociology.yaml     |   5 +
 ...global_mmlu_full_uk_us_foreign_policy.yaml |   5 +
 .../full/uk/global_mmlu_full_uk_virology.yaml |   5 +
 .../global_mmlu_full_uk_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/uk/utils.py  |  73 +++++
 .../full/vi/_global_mmlu_full_vi.yaml         |  11 +
 .../vi/_global_mmlu_full_vi_humanities.yaml   |   8 +
 .../full/vi/_global_mmlu_full_vi_other.yaml   |   8 +
 .../_global_mmlu_full_vi_social_sciences.yaml |   8 +
 .../full/vi/_global_mmlu_full_vi_stem.yaml    |   8 +
 .../global_mmlu/full/vi/_vi_template_yaml     |  16 ++
 .../global_mmlu_full_vi_abstract_algebra.yaml |   5 +
 .../full/vi/global_mmlu_full_vi_anatomy.yaml  |   5 +
 .../vi/global_mmlu_full_vi_astronomy.yaml     |   5 +
 .../global_mmlu_full_vi_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_vi_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_vi_college_biology.yaml  |   5 +
 ...global_mmlu_full_vi_college_chemistry.yaml |   5 +
 ...mmlu_full_vi_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_vi_college_mathematics.yaml |   5 +
 .../global_mmlu_full_vi_college_medicine.yaml |   5 +
 .../global_mmlu_full_vi_college_physics.yaml  |   5 +
 ...global_mmlu_full_vi_computer_security.yaml |   5 +
 ...lobal_mmlu_full_vi_conceptual_physics.yaml |   5 +
 .../vi/global_mmlu_full_vi_econometrics.yaml  |   5 +
 ...l_mmlu_full_vi_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_vi_elementary_mathematics.yaml |   5 +
 .../vi/global_mmlu_full_vi_formal_logic.yaml  |   5 +
 .../vi/global_mmlu_full_vi_global_facts.yaml  |   5 +
 ...obal_mmlu_full_vi_high_school_biology.yaml |   5 +
 ...al_mmlu_full_vi_high_school_chemistry.yaml |   5 +
 ..._full_vi_high_school_computer_science.yaml |   5 +
 ..._full_vi_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_vi_high_school_geography.yaml |   5 +
 ...i_high_school_government_and_politics.yaml |   5 +
 ...lu_full_vi_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_vi_high_school_mathematics.yaml |   5 +
 ...lu_full_vi_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_vi_high_school_physics.yaml |   5 +
 ...l_mmlu_full_vi_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_vi_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_vi_high_school_us_history.yaml |   5 +
 ...mlu_full_vi_high_school_world_history.yaml |   5 +
 .../vi/global_mmlu_full_vi_human_aging.yaml   |   5 +
 .../global_mmlu_full_vi_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_vi_international_law.yaml |   5 +
 .../vi/global_mmlu_full_vi_jurisprudence.yaml |   5 +
 ...global_mmlu_full_vi_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_vi_machine_learning.yaml |   5 +
 .../vi/global_mmlu_full_vi_management.yaml    |   5 +
 .../vi/global_mmlu_full_vi_marketing.yaml     |   5 +
 .../global_mmlu_full_vi_medical_genetics.yaml |   5 +
 .../vi/global_mmlu_full_vi_miscellaneous.yaml |   5 +
 .../global_mmlu_full_vi_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_vi_moral_scenarios.yaml  |   5 +
 .../vi/global_mmlu_full_vi_nutrition.yaml     |   5 +
 .../vi/global_mmlu_full_vi_philosophy.yaml    |   5 +
 .../vi/global_mmlu_full_vi_prehistory.yaml    |   5 +
 ..._mmlu_full_vi_professional_accounting.yaml |   5 +
 .../global_mmlu_full_vi_professional_law.yaml |   5 +
 ...al_mmlu_full_vi_professional_medicine.yaml |   5 +
 ..._mmlu_full_vi_professional_psychology.yaml |   5 +
 .../global_mmlu_full_vi_public_relations.yaml |   5 +
 .../global_mmlu_full_vi_security_studies.yaml |   5 +
 .../vi/global_mmlu_full_vi_sociology.yaml     |   5 +
 ...global_mmlu_full_vi_us_foreign_policy.yaml |   5 +
 .../full/vi/global_mmlu_full_vi_virology.yaml |   5 +
 .../global_mmlu_full_vi_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/vi/utils.py  |  73 +++++
 .../full/yo/_global_mmlu_full_yo.yaml         |  11 +
 .../yo/_global_mmlu_full_yo_humanities.yaml   |   8 +
 .../full/yo/_global_mmlu_full_yo_other.yaml   |   8 +
 .../_global_mmlu_full_yo_social_sciences.yaml |   8 +
 .../full/yo/_global_mmlu_full_yo_stem.yaml    |   8 +
 .../global_mmlu/full/yo/_yo_template_yaml     |  16 ++
 .../global_mmlu_full_yo_abstract_algebra.yaml |   5 +
 .../full/yo/global_mmlu_full_yo_anatomy.yaml  |   5 +
 .../yo/global_mmlu_full_yo_astronomy.yaml     |   5 +
 .../global_mmlu_full_yo_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_yo_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_yo_college_biology.yaml  |   5 +
 ...global_mmlu_full_yo_college_chemistry.yaml |   5 +
 ...mmlu_full_yo_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_yo_college_mathematics.yaml |   5 +
 .../global_mmlu_full_yo_college_medicine.yaml |   5 +
 .../global_mmlu_full_yo_college_physics.yaml  |   5 +
 ...global_mmlu_full_yo_computer_security.yaml |   5 +
 ...lobal_mmlu_full_yo_conceptual_physics.yaml |   5 +
 .../yo/global_mmlu_full_yo_econometrics.yaml  |   5 +
 ...l_mmlu_full_yo_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_yo_elementary_mathematics.yaml |   5 +
 .../yo/global_mmlu_full_yo_formal_logic.yaml  |   5 +
 .../yo/global_mmlu_full_yo_global_facts.yaml  |   5 +
 ...obal_mmlu_full_yo_high_school_biology.yaml |   5 +
 ...al_mmlu_full_yo_high_school_chemistry.yaml |   5 +
 ..._full_yo_high_school_computer_science.yaml |   5 +
 ..._full_yo_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_yo_high_school_geography.yaml |   5 +
 ...o_high_school_government_and_politics.yaml |   5 +
 ...lu_full_yo_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_yo_high_school_mathematics.yaml |   5 +
 ...lu_full_yo_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_yo_high_school_physics.yaml |   5 +
 ...l_mmlu_full_yo_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_yo_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_yo_high_school_us_history.yaml |   5 +
 ...mlu_full_yo_high_school_world_history.yaml |   5 +
 .../yo/global_mmlu_full_yo_human_aging.yaml   |   5 +
 .../global_mmlu_full_yo_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_yo_international_law.yaml |   5 +
 .../yo/global_mmlu_full_yo_jurisprudence.yaml |   5 +
 ...global_mmlu_full_yo_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_yo_machine_learning.yaml |   5 +
 .../yo/global_mmlu_full_yo_management.yaml    |   5 +
 .../yo/global_mmlu_full_yo_marketing.yaml     |   5 +
 .../global_mmlu_full_yo_medical_genetics.yaml |   5 +
 .../yo/global_mmlu_full_yo_miscellaneous.yaml |   5 +
 .../global_mmlu_full_yo_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_yo_moral_scenarios.yaml  |   5 +
 .../yo/global_mmlu_full_yo_nutrition.yaml     |   5 +
 .../yo/global_mmlu_full_yo_philosophy.yaml    |   5 +
 .../yo/global_mmlu_full_yo_prehistory.yaml    |   5 +
 ..._mmlu_full_yo_professional_accounting.yaml |   5 +
 .../global_mmlu_full_yo_professional_law.yaml |   5 +
 ...al_mmlu_full_yo_professional_medicine.yaml |   5 +
 ..._mmlu_full_yo_professional_psychology.yaml |   5 +
 .../global_mmlu_full_yo_public_relations.yaml |   5 +
 .../global_mmlu_full_yo_security_studies.yaml |   5 +
 .../yo/global_mmlu_full_yo_sociology.yaml     |   5 +
 ...global_mmlu_full_yo_us_foreign_policy.yaml |   5 +
 .../full/yo/global_mmlu_full_yo_virology.yaml |   5 +
 .../global_mmlu_full_yo_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/yo/utils.py  |  73 +++++
 .../full/zh/_global_mmlu_full_zh.yaml         |  11 +
 .../zh/_global_mmlu_full_zh_humanities.yaml   |   8 +
 .../full/zh/_global_mmlu_full_zh_other.yaml   |   8 +
 .../_global_mmlu_full_zh_social_sciences.yaml |   8 +
 .../full/zh/_global_mmlu_full_zh_stem.yaml    |   8 +
 .../global_mmlu/full/zh/_zh_template_yaml     |  16 ++
 .../global_mmlu_full_zh_abstract_algebra.yaml |   5 +
 .../full/zh/global_mmlu_full_zh_anatomy.yaml  |   5 +
 .../zh/global_mmlu_full_zh_astronomy.yaml     |   5 +
 .../global_mmlu_full_zh_business_ethics.yaml  |   5 +
 ...lobal_mmlu_full_zh_clinical_knowledge.yaml |   5 +
 .../global_mmlu_full_zh_college_biology.yaml  |   5 +
 ...global_mmlu_full_zh_college_chemistry.yaml |   5 +
 ...mmlu_full_zh_college_computer_science.yaml |   5 +
 ...obal_mmlu_full_zh_college_mathematics.yaml |   5 +
 .../global_mmlu_full_zh_college_medicine.yaml |   5 +
 .../global_mmlu_full_zh_college_physics.yaml  |   5 +
 ...global_mmlu_full_zh_computer_security.yaml |   5 +
 ...lobal_mmlu_full_zh_conceptual_physics.yaml |   5 +
 .../zh/global_mmlu_full_zh_econometrics.yaml  |   5 +
 ...l_mmlu_full_zh_electrical_engineering.yaml |   5 +
 ...l_mmlu_full_zh_elementary_mathematics.yaml |   5 +
 .../zh/global_mmlu_full_zh_formal_logic.yaml  |   5 +
 .../zh/global_mmlu_full_zh_global_facts.yaml  |   5 +
 ...obal_mmlu_full_zh_high_school_biology.yaml |   5 +
 ...al_mmlu_full_zh_high_school_chemistry.yaml |   5 +
 ..._full_zh_high_school_computer_science.yaml |   5 +
 ..._full_zh_high_school_european_history.yaml |   5 +
 ...al_mmlu_full_zh_high_school_geography.yaml |   5 +
 ...h_high_school_government_and_politics.yaml |   5 +
 ...lu_full_zh_high_school_macroeconomics.yaml |   5 +
 ..._mmlu_full_zh_high_school_mathematics.yaml |   5 +
 ...lu_full_zh_high_school_microeconomics.yaml |   5 +
 ...obal_mmlu_full_zh_high_school_physics.yaml |   5 +
 ...l_mmlu_full_zh_high_school_psychology.yaml |   5 +
 ...l_mmlu_full_zh_high_school_statistics.yaml |   5 +
 ...l_mmlu_full_zh_high_school_us_history.yaml |   5 +
 ...mlu_full_zh_high_school_world_history.yaml |   5 +
 .../zh/global_mmlu_full_zh_human_aging.yaml   |   5 +
 .../global_mmlu_full_zh_human_sexuality.yaml  |   5 +
 ...global_mmlu_full_zh_international_law.yaml |   5 +
 .../zh/global_mmlu_full_zh_jurisprudence.yaml |   5 +
 ...global_mmlu_full_zh_logical_fallacies.yaml |   5 +
 .../global_mmlu_full_zh_machine_learning.yaml |   5 +
 .../zh/global_mmlu_full_zh_management.yaml    |   5 +
 .../zh/global_mmlu_full_zh_marketing.yaml     |   5 +
 .../global_mmlu_full_zh_medical_genetics.yaml |   5 +
 .../zh/global_mmlu_full_zh_miscellaneous.yaml |   5 +
 .../global_mmlu_full_zh_moral_disputes.yaml   |   5 +
 .../global_mmlu_full_zh_moral_scenarios.yaml  |   5 +
 .../zh/global_mmlu_full_zh_nutrition.yaml     |   5 +
 .../zh/global_mmlu_full_zh_philosophy.yaml    |   5 +
 .../zh/global_mmlu_full_zh_prehistory.yaml    |   5 +
 ..._mmlu_full_zh_professional_accounting.yaml |   5 +
 .../global_mmlu_full_zh_professional_law.yaml |   5 +
 ...al_mmlu_full_zh_professional_medicine.yaml |   5 +
 ..._mmlu_full_zh_professional_psychology.yaml |   5 +
 .../global_mmlu_full_zh_public_relations.yaml |   5 +
 .../global_mmlu_full_zh_security_studies.yaml |   5 +
 .../zh/global_mmlu_full_zh_sociology.yaml     |   5 +
 ...global_mmlu_full_zh_us_foreign_policy.yaml |   5 +
 .../full/zh/global_mmlu_full_zh_virology.yaml |   5 +
 .../global_mmlu_full_zh_world_religions.yaml  |   5 +
 lmms_eval/tasks/global_mmlu/full/zh/utils.py  |  73 +++++
 lmms_eval/utils.py                            |   5 +-
 2854 files changed, 18713 insertions(+), 1021 deletions(-)
 create mode 100644 lmms_eval/tasks/alm_bench/_default_template_yaml
 delete mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml
 delete mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml
 delete mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-eng.yaml
 delete mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml
 delete mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml
 delete mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml
 delete mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml
 delete mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml
 delete mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml
 delete mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml
 delete mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml
 delete mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen.yaml
 delete mode 100644 lmms_eval/tasks/ayavisionbench/eval_judge_template.yaml
 delete mode 100644 lmms_eval/tasks/ayavisionbench/judge_eval.py
 delete mode 100755 lmms_eval/tasks/ayavisionbench/judge_templates.py
 delete mode 100644 lmms_eval/tasks/ayavisionbench/judge_utils.py
 delete mode 100644 lmms_eval/tasks/ayavisionbench/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/README.md
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ar/_ar_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ar/_global_mmlu_ar.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_business.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_medical.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ar/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/default/bn/_bn_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/bn/_global_mmlu_bn.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_business.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_medical.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/bn/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/default/de/_de_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/de/_global_mmlu_de.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_business.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_medical.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/de/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/default/en/_en_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/en/_global_mmlu_en.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_business.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_medical.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/en/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/default/es/_es_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/es/_global_mmlu_es.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_business.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_medical.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/es/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/default/fr/_fr_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/fr/_global_mmlu_fr.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_business.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_medical.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/fr/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/default/hi/_global_mmlu_hi.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/hi/_hi_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_business.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_medical.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/hi/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/default/id/_global_mmlu_id.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/id/_id_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_business.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_medical.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/id/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/default/it/_global_mmlu_it.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/it/_it_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_business.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_medical.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/it/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ja/_global_mmlu_ja.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ja/_ja_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_business.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_medical.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ja/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ko/_global_mmlu_ko.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ko/_ko_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_business.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_medical.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/ko/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/default/pt/_global_mmlu_pt.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/pt/_pt_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_business.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_medical.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/pt/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/default/sw/_global_mmlu_sw.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/sw/_sw_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_business.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_medical.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/sw/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/default/yo/_global_mmlu_yo.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/yo/_yo_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_business.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_medical.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/yo/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/default/zh/_global_mmlu_zh.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/zh/_zh_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_business.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_medical.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/default/zh/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/_am_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/_global_mmlu_full_am.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/_global_mmlu_full_am_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/_global_mmlu_full_am_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/_global_mmlu_full_am_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/_global_mmlu_full_am_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/am/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/_ar_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/_global_mmlu_full_ar.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/_global_mmlu_full_ar_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/_global_mmlu_full_ar_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/_global_mmlu_full_ar_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/_global_mmlu_full_ar_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ar/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/_bn_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/_global_mmlu_full_bn.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/bn/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/_cs_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/cs/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/_de_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/_global_mmlu_full_de.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/_global_mmlu_full_de_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/_global_mmlu_full_de_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/_global_mmlu_full_de_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/_global_mmlu_full_de_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/de/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/_el_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/el/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/_en_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/en/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/_es_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/_global_mmlu_full_es.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/_global_mmlu_full_es_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/_global_mmlu_full_es_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/_global_mmlu_full_es_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/_global_mmlu_full_es_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/es/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/_fa_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fa/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/_fil_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fil/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/_fr_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/fr/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/_ha_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ha/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/_he_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/he/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/_hi_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/hi/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/_id_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/id/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/_ig_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ig/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/_it_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/it/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/_ja_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ja/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/_ko_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ko/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/_ky_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ky/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/_lt_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/lt/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/_mg_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/mg/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/_ms_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ms/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/_ne_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ne/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/_nl_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/nl/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/_ny_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ny/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/_pl_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pl/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/_pt_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/pt/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/_ro_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ro/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/_ru_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/ru/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/_si_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/si/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/_sn_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sn/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/_so_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/so/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/_sr_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sr/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/_sv_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sv/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/_global_mmlu_full_sw.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/_sw_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/sw/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/_global_mmlu_full_te.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/_global_mmlu_full_te_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/_global_mmlu_full_te_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/_global_mmlu_full_te_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/_global_mmlu_full_te_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/_te_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/te/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/_tr_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/tr/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/_global_mmlu_full_uk.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/_uk_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/uk/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/_global_mmlu_full_vi.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/_vi_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/vi/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/_global_mmlu_full_yo.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/_yo_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/yo/utils.py
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_humanities.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_other.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_social_sciences.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_stem.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/_zh_template_yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_abstract_algebra.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_anatomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_astronomy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_business_ethics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_clinical_knowledge.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_computer_security.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_conceptual_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_econometrics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_electrical_engineering.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_elementary_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_formal_logic.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_global_facts.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_biology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_chemistry.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_computer_science.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_european_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_geography.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_government_and_politics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_macroeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_mathematics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_microeconomics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_physics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_statistics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_us_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_world_history.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_human_aging.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_human_sexuality.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_international_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_jurisprudence.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_logical_fallacies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_machine_learning.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_management.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_marketing.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_medical_genetics.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_miscellaneous.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_moral_disputes.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_moral_scenarios.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_nutrition.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_philosophy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_prehistory.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_accounting.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_law.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_medicine.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_psychology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_public_relations.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_security_studies.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_sociology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_us_foreign_policy.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_virology.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_world_religions.yaml
 create mode 100644 lmms_eval/tasks/global_mmlu/full/zh/utils.py

diff --git a/lmms_eval/models/llava.py b/lmms_eval/models/llava.py
index f47839e92..dd130ba2a 100755
--- a/lmms_eval/models/llava.py
+++ b/lmms_eval/models/llava.py
@@ -249,8 +249,8 @@ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
                 conv = copy.deepcopy(conv_templates[self.conv_template])
             else:
                 conv = conv_templates[self.conv_template].copy()
-            if self.system_prompt and conv.system:
-                conv.system = self.system_prompt
+            if self.add_system_prompt and conv.system:
+                conv.system = self.add_system_prompt
             conv.append_message(conv.roles[0], prompts_input)
             conv.append_message(conv.roles[1], None)
             prompt = conv.get_prompt()
@@ -314,6 +314,7 @@ def _collate(x):
         chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
         num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
         pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
+        breakpoint()
         for chunk in chunks:
             contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
             task = task[0]
@@ -374,12 +375,12 @@ def _collate(x):
                 # if conv.system:
                 #     conv.system = self.add_system_prompt
                 if self.add_system_prompt and conv.system:
-                    conv.system = self.system_prompt
+                    conv.system = self.add_system_prompt
                 conv.append_message(conv.roles[0], question)
                 conv.append_message(conv.roles[1], None)
                 prompt_question = conv.get_prompt()
                 question_input.append(prompt_question)
-
+            
             # input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
             # preconfigure gen_kwargs with defaults
             gen_kwargs["image_sizes"] = [flattened_visuals[idx].size for idx in range(len(flattened_visuals))]
diff --git a/lmms_eval/tasks/alm_bench/_default_template_yaml b/lmms_eval/tasks/alm_bench/_default_template_yaml
new file mode 100644
index 000000000..4a86a897a
--- /dev/null
+++ b/lmms_eval/tasks/alm_bench/_default_template_yaml
@@ -0,0 +1,29 @@
+dataset_path: sonalsannigrahi/alm-bench-mcq-subset
+task: "alm-bench"
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.alm_bench_doc_to_visual
+doc_to_text: !function utils.alm_bench_doc_to_text
+doc_to_target: !function utils.alm_bench_doc_to_target
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  do_sample: False
+process_results: !function utils.alm_bench_process_results
+metric_list:
+  - metric: results
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  - version: 0.0
+model_specific_target_kwargs:
+  default: "mcq"
+lmms_eval_specific_kwargs:
+  default:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    # pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    post_prompt: ""
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-de.yaml b/lmms_eval/tasks/alm_bench/alm-bench-de.yaml
index 02c198b9b..4d6ce6884 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-de.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-de.yaml
@@ -6,9 +6,10 @@ doc_to_visual: !function utils.alm_bench_doc_to_visual
 doc_to_text: !function utils.alm_bench_doc_to_text
 doc_to_target: !function utils.alm_bench_doc_to_target
 generation_kwargs:
-  max_new_tokens: 128
+  max_new_tokens: 1024
   temperature: 0
   do_sample: False
+process_results: !function utils.alm_bench_process_results
 metric_list:
   - metric: exact_match
     aggregation: mean
@@ -20,8 +21,9 @@ model_specific_target_kwargs:
 lmms_eval_specific_kwargs:
   default:
     prompt_format: "mcq"
-    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    # pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-en.yaml b/lmms_eval/tasks/alm_bench/alm-bench-en.yaml
index fcb767166..0c3e4f424 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-en.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-en.yaml
@@ -6,9 +6,10 @@ doc_to_visual: !function utils.alm_bench_doc_to_visual
 doc_to_text: !function utils.alm_bench_doc_to_text
 doc_to_target: !function utils.alm_bench_doc_to_target
 generation_kwargs:
-  max_new_tokens: 128
+  max_new_tokens: 1024
   temperature: 0
   do_sample: False
+process_results: !function utils.alm_bench_process_results
 metric_list:
   - metric: exact_match
     aggregation: mean
@@ -20,7 +21,8 @@ model_specific_target_kwargs:
 lmms_eval_specific_kwargs:
   default:
     prompt_format: "mcq"
-    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    # pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-es.yaml b/lmms_eval/tasks/alm_bench/alm-bench-es.yaml
index 423730d79..a91464679 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-es.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-es.yaml
@@ -6,9 +6,10 @@ doc_to_visual: !function utils.alm_bench_doc_to_visual
 doc_to_text: !function utils.alm_bench_doc_to_text
 doc_to_target: !function utils.alm_bench_doc_to_target
 generation_kwargs:
-  max_new_tokens: 128
+  max_new_tokens: 1024
   temperature: 0
   do_sample: False
+process_results: !function utils.alm_bench_process_results
 metric_list:
   - metric: exact_match
     aggregation: mean
@@ -20,8 +21,9 @@ model_specific_target_kwargs:
 lmms_eval_specific_kwargs:
   default:
     prompt_format: "mcq"
-    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    # pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-fr.yaml b/lmms_eval/tasks/alm_bench/alm-bench-fr.yaml
index 24947b8e9..b3eff6fee 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-fr.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-fr.yaml
@@ -6,9 +6,10 @@ doc_to_visual: !function utils.alm_bench_doc_to_visual
 doc_to_text: !function utils.alm_bench_doc_to_text
 doc_to_target: !function utils.alm_bench_doc_to_target
 generation_kwargs:
-  max_new_tokens: 128
+  max_new_tokens: 1024
   temperature: 0
   do_sample: False
+process_results: !function utils.alm_bench_process_results
 metric_list:
   - metric: exact_match
     aggregation: mean
@@ -20,8 +21,9 @@ model_specific_target_kwargs:
 lmms_eval_specific_kwargs:
   default:
     prompt_format: "mcq"
-    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    # pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-it.yaml b/lmms_eval/tasks/alm_bench/alm-bench-it.yaml
index 956089c1d..912dd237c 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-it.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-it.yaml
@@ -6,9 +6,10 @@ doc_to_visual: !function utils.alm_bench_doc_to_visual
 doc_to_text: !function utils.alm_bench_doc_to_text
 doc_to_target: !function utils.alm_bench_doc_to_target
 generation_kwargs:
-  max_new_tokens: 128
+  max_new_tokens: 1024
   temperature: 0
   do_sample: False
+process_results: !function utils.alm_bench_process_results
 metric_list:
   - metric: exact_match
     aggregation: mean
@@ -20,8 +21,9 @@ model_specific_target_kwargs:
 lmms_eval_specific_kwargs:
   default:
     prompt_format: "mcq"
-    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    # pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-ko.yaml b/lmms_eval/tasks/alm_bench/alm-bench-ko.yaml
index ad13c1e62..e3bd957a6 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-ko.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-ko.yaml
@@ -6,9 +6,10 @@ doc_to_visual: !function utils.alm_bench_doc_to_visual
 doc_to_text: !function utils.alm_bench_doc_to_text
 doc_to_target: !function utils.alm_bench_doc_to_target
 generation_kwargs:
-  max_new_tokens: 128
+  max_new_tokens: 1024
   temperature: 0
   do_sample: False
+process_results: !function utils.alm_bench_process_results
 metric_list:
   - metric: exact_match
     aggregation: mean
@@ -20,8 +21,9 @@ model_specific_target_kwargs:
 lmms_eval_specific_kwargs:
   default:
     prompt_format: "mcq"
-    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    # pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-nl.yaml b/lmms_eval/tasks/alm_bench/alm-bench-nl.yaml
index 271d72b04..4d8bf3c3f 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-nl.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-nl.yaml
@@ -6,9 +6,10 @@ doc_to_visual: !function utils.alm_bench_doc_to_visual
 doc_to_text: !function utils.alm_bench_doc_to_text
 doc_to_target: !function utils.alm_bench_doc_to_target
 generation_kwargs:
-  max_new_tokens: 128
+  max_new_tokens: 1024
   temperature: 0
   do_sample: False
+process_results: !function utils.alm_bench_process_results
 metric_list:
   - metric: exact_match
     aggregation: mean
@@ -20,8 +21,9 @@ model_specific_target_kwargs:
 lmms_eval_specific_kwargs:
   default:
     prompt_format: "mcq"
-    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    # pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-pt.yaml b/lmms_eval/tasks/alm_bench/alm-bench-pt.yaml
index bf453f135..70623a2f7 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-pt.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-pt.yaml
@@ -6,9 +6,10 @@ doc_to_visual: !function utils.alm_bench_doc_to_visual
 doc_to_text: !function utils.alm_bench_doc_to_text
 doc_to_target: !function utils.alm_bench_doc_to_target
 generation_kwargs:
-  max_new_tokens: 128
+  max_new_tokens: 1024
   temperature: 0
   do_sample: False
+process_results: !function utils.alm_bench_process_results
 metric_list:
   - metric: exact_match
     aggregation: mean
@@ -20,8 +21,9 @@ model_specific_target_kwargs:
 lmms_eval_specific_kwargs:
   default:
     prompt_format: "mcq"
-    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    # pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-ru.yaml b/lmms_eval/tasks/alm_bench/alm-bench-ru.yaml
index 4f9ec6fa6..5adcf6ac2 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-ru.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-ru.yaml
@@ -6,9 +6,10 @@ doc_to_visual: !function utils.alm_bench_doc_to_visual
 doc_to_text: !function utils.alm_bench_doc_to_text
 doc_to_target: !function utils.alm_bench_doc_to_target
 generation_kwargs:
-  max_new_tokens: 128
+  max_new_tokens: 1024
   temperature: 0
   do_sample: False
+process_results: !function utils.alm_bench_process_results
 metric_list:
   - metric: exact_match
     aggregation: mean
@@ -20,8 +21,9 @@ model_specific_target_kwargs:
 lmms_eval_specific_kwargs:
   default:
     prompt_format: "mcq"
-    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. Strictly answer the choice only. No additional text."
+    # pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
+    post_prompt: " Answer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/utils.py b/lmms_eval/tasks/alm_bench/utils.py
index 0d0a8adf3..008a1724c 100644
--- a/lmms_eval/tasks/alm_bench/utils.py
+++ b/lmms_eval/tasks/alm_bench/utils.py
@@ -51,7 +51,7 @@ def alm_bench_process_results(doc, results):
     if target == None:
         print(doc["Translated_Answer"])
     match = exact_match(pred, target.strip("."))
-    return {"match": match}
+    return {"exact_match": match}
 
 
 def alm_bench_doc_to_target(doc, model_specific_target_kwargs):
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml
deleted file mode 100644
index d35aca642..000000000
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml
+++ /dev/null
@@ -1,34 +0,0 @@
-dataset_path: CohereForAI/AyaVisionBench
-dataset_name: ces_Latn
-task: aya-vision-bench-gen-cs
-test_split: test
-output_type: generate_until
-process_docs: !function utils.process_docs
-doc_to_visual: !function utils.gen_doc_to_visual
-doc_to_text: !function utils.gen_doc_to_text
-doc_to_target: ""
-process_results: !function utils.gen_process_results
-
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-# metric_list:
-#   - metric: results
-#     aggregation: !function utils.aggregate_results
-
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: ""
-    post_prompt: "Answer:"
-
-
-
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml
deleted file mode 100644
index 043af92fb..000000000
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml
+++ /dev/null
@@ -1,34 +0,0 @@
-dataset_path: CohereForAI/AyaVisionBench
-dataset_name: deu_Latn
-task: aya-vision-bench-gen-de
-test_split: test
-output_type: generate_until
-process_docs: !function utils.process_docs
-doc_to_visual: !function utils.gen_doc_to_visual
-doc_to_text: !function utils.gen_doc_to_text
-doc_to_target: ""
-process_results: !function utils.gen_process_results
-
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-# metric_list:
-#   - metric: results
-#     aggregation: !function utils.aggregate_results
-
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: ""
-    post_prompt: "Answer:"
-
-
-
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-eng.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-eng.yaml
deleted file mode 100644
index 40a427bc8..000000000
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-eng.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
-dataset_path: CohereForAI/AyaVisionBench
-dataset_name: eng_Latn
-task: aya-vision-bench-gen-eng
-test_split: test
-output_type: generate_until
-
-process_docs: !function utils.process_docs
-doc_to_visual: !function utils.gen_doc_to_visual
-doc_to_text: !function utils.gen_doc_to_text
-doc_to_target: ""
-process_results: !function utils.gen_process_results
-
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-# metric_list:
-#   - metric: results
-#     aggregation: !function utils.aggregate_results
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: ""
-    post_prompt: "Answer:"
-
-
-
-
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml
deleted file mode 100644
index df13935b1..000000000
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml
+++ /dev/null
@@ -1,34 +0,0 @@
-dataset_path: CohereForAI/AyaVisionBench
-dataset_name: spa_Latn
-task: aya-vision-bench-gen-es
-test_split: test
-output_type: generate_until
-
-process_docs: !function utils.process_docs
-doc_to_visual: !function utils.gen_doc_to_visual
-doc_to_text: !function utils.gen_doc_to_text
-doc_to_target: ""
-process_results: !function utils.gen_process_results
-
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-# metric_list:
-#   - metric: results
-#     aggregation: !function utils.aggregate_results
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: ""
-    post_prompt: "Answer:"
-
-
-
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml
deleted file mode 100644
index 3e624eef5..000000000
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
-dataset_path: CohereForAI/AyaVisionBench
-dataset_name: fra_Latn
-task: aya-vision-bench-gen-fr
-test_split: test
-output_type: generate_until
-
-process_docs: !function utils.process_docs
-doc_to_visual: !function utils.gen_doc_to_visual
-doc_to_text: !function utils.gen_doc_to_text
-doc_to_target: ""
-process_results: !function utils.gen_process_results
-
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-# metric_list:
-#   - metric: results
-#     aggregation: !function utils.aggregate_results
-
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: ""
-    post_prompt: "Answer:"
-
-
-
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml
deleted file mode 100644
index 110e4fc7b..000000000
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
-dataset_path: CohereForAI/AyaVisionBench
-dataset_name: ita_Latn
-task: aya-vision-bench-gen-it
-test_split: test
-output_type: generate_until
-
-process_docs: !function utils.process_docs
-doc_to_visual: !function utils.gen_doc_to_visual
-doc_to_text: !function utils.gen_doc_to_text
-doc_to_target: ""
-process_results: !function utils.gen_process_results
-
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-# metric_list:
-#   - metric: results
-#     aggregation: !function utils.aggregate_results
-
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: ""
-    post_prompt: "Answer:"
-
-
-
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml
deleted file mode 100644
index 12058e158..000000000
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
-dataset_path: CohereForAI/AyaVisionBench
-dataset_name: kor_Hang
-task: aya-vision-bench-gen-ko
-test_split: test
-output_type: generate_until
-
-process_docs: !function utils.process_docs
-doc_to_visual: !function utils.gen_doc_to_visual
-doc_to_text: !function utils.gen_doc_to_text
-doc_to_target: ""
-process_results: !function utils.gen_process_results
-
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-# metric_list:
-#   - metric: results
-#     aggregation: !function utils.aggregate_results
-
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: ""
-    post_prompt: "Answer:"
-
-
-
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml
deleted file mode 100644
index c8b1f2c88..000000000
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
-dataset_path: CohereForAI/AyaVisionBench
-dataset_name: nld_Latn
-task: aya-vision-bench-gen-nl
-test_split: test
-output_type: generate_until
-
-process_docs: !function utils.process_docs
-doc_to_visual: !function utils.gen_doc_to_visual
-doc_to_text: !function utils.gen_doc_to_text
-doc_to_target: ""
-process_results: !function utils.gen_process_results
-
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-# metric_list:
-#   - metric: results
-#     aggregation: !function utils.aggregate_results
-
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: ""
-    post_prompt: "Answer:"
-
-
-
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml
deleted file mode 100644
index 96b8c8952..000000000
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
-dataset_path: CohereForAI/AyaVisionBench
-dataset_name: por_Latn
-task: aya-vision-bench-gen-pt
-test_split: test
-output_type: generate_until
-
-process_docs: !function utils.process_docs
-doc_to_visual: !function utils.gen_doc_to_visual
-doc_to_text: !function utils.gen_doc_to_text
-doc_to_target: ""
-process_results: !function utils.gen_process_results
-
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-# metric_list:
-#   - metric: results
-#     aggregation: !function utils.aggregate_results
-
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: ""
-    post_prompt: "Answer:"
-
-
-
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml
deleted file mode 100644
index 8eabc0b1b..000000000
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
-dataset_path: CohereForAI/AyaVisionBench
-dataset_name: rus_Cyrl
-task: aya-vision-bench-gen-ru
-test_split: test
-output_type: generate_until
-
-process_docs: !function utils.process_docs
-doc_to_visual: !function utils.gen_doc_to_visual
-doc_to_text: !function utils.gen_doc_to_text
-doc_to_target: ""
-process_results: !function utils.gen_process_results
-
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-# metric_list:
-#   - metric: results
-#     aggregation: !function utils.aggregate_results
-
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: ""
-    post_prompt: "Answer:"
-
-
-
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml
deleted file mode 100644
index 9c5093878..000000000
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
-dataset_path: CohereForAI/AyaVisionBench
-dataset_name: zho_Hans
-task: aya-vision-bench-gen-zh
-test_split: test
-output_type: generate_until
-
-process_docs: !function utils.process_docs
-doc_to_visual: !function utils.gen_doc_to_visual
-doc_to_text: !function utils.gen_doc_to_text
-doc_to_target: ""
-process_results: !function utils.gen_process_results
-
-
-generation_kwargs:
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-
-# metric_list:
-#   - metric: results
-#     aggregation: !function utils.aggregate_results
-
-
-metadata:
-  - version: 0.0
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: ""
-    post_prompt: "Answer:"
-
-
-
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen.yaml
deleted file mode 100644
index ea6402f48..000000000
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-group: aya-vision-bench-gen
-task:
-  - aya-vision-bench-gen-eng
-  - aya-vision-bench-gen-de
-  - aya-vision-bench-gen-es
-  - aya-vision-bench-gen-fr
-  - aya-vision-bench-gen-it
-  - aya-vision-bench-gen-ko
-  - aya-vision-bench-gen-zh
-  - aya-vision-bench-gen-nl
-  - aya-vision-bench-gen-pt
-  - aya-vision-bench-gen-ru
-  # - aya-vision-bench-gen-cs
-
diff --git a/lmms_eval/tasks/ayavisionbench/eval_judge_template.yaml b/lmms_eval/tasks/ayavisionbench/eval_judge_template.yaml
deleted file mode 100644
index fcf0e0a8a..000000000
--- a/lmms_eval/tasks/ayavisionbench/eval_judge_template.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-# if comparative, then the judge will compare the baseline model output with the predicted outputs. If the baseline model outputs path is not found, then the predicted outputs will be just saved, without running the judge.
-# if direct assessment, then the judge will just judge the predicted outputs!
-# to disable the judge, set run_judge to false
-# run_judge: false
-# api_type: openai #openai #anthropic
-# judge_model_name: gpt-4o-mini  #gpt-4o #neulab/claude-3-7-sonnet-20250219  #claude-3-7-sonnet-20250219
-# judge_prompt_type: comparative
-# text_only: false
-# baseline_model_outputs_path: /mnt/scratch-artemis/manos/data/tower-vision-eval-outputs/tower-vision/tower-vision-debug-results/aya-vision-bench-gen-eng/Unbabel__lnext-qwen2p5-7b-siglip2-v5/Unbabel__lnext-qwen2p5-7b-siglip2-v5/20250418_015501_samples_aya-vision-bench-gen-eng.jsonl #/mnt/scratch-artemis/manos/data/tower-vision-eval-outputs/tower-vision/tower-vision-debug-results/aya-vision-bench-gen-eng/Unbabel__lnext-qwen2p5-7b-siglip2-v5/Unbabel__lnext-qwen2p5-7b-siglip2-v5/20250416_060306_samples_aya-vision-bench-gen-eng.jsonl
-# # max_tokens: 2048
-# temperature: 0.0
-# top_p: 1.0
-# tensor_parallel_size: 1
-# save_judge_parsed_outputs: true
-# api_url: https://api.openai.com/v1/chat/completions    #https://api.anthropic.com/v1/messages  for anthropic
-# api_url: https://cmu.litellm.ai # https://api.openai.com/v1/chat/completions
diff --git a/lmms_eval/tasks/ayavisionbench/judge_eval.py b/lmms_eval/tasks/ayavisionbench/judge_eval.py
deleted file mode 100644
index 1d19701f7..000000000
--- a/lmms_eval/tasks/ayavisionbench/judge_eval.py
+++ /dev/null
@@ -1,151 +0,0 @@
-import argparse
-import json
-from PIL import Image
-import os
-import io
-from loguru import logger
-
-from datasets import load_dataset
-from lmms_eval.tasks.ayavisionbench.judge_utils import run_judge,compute_results,parse_judge_responses
-
-def load_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--api_type",choices=["openai", "anthropic","litellm"], type=str, required=True)
-    parser.add_argument("--judge_model_name",default="gpt-4o-mini", type=str, required=True)
-    parser.add_argument("--judge_prompt_type",default="comparative",choices=["comparative", "direct assessment"], type=str, required=True)
-    parser.add_argument("--text_only",default=False, type=bool, required=True,help="if true, the judge model will not see the image.")
-    parser.add_argument("--model_output_path", type=str, required=True)
-    parser.add_argument("--baseline_output_path", type=str, required=True)
-    parser.add_argument("--lp", type=str, required=True)
-    parser.add_argument("--output_dir", type=str, required=True)
-    parser.add_argument("--save_judge_parsed_outputs",default=True, type=bool, required=True,help="if true, the judge parsed outputs will be saved.")
-    parser.add_argument("--save_judge_full_responses",default=True, type=bool, required=True,help="if true, the judge full responses will be saved.")
-    parser.add_argument("--save_judge_results",default=True, type=bool, required=True,help="if true, the judge results will be saved.")
-    parser.add_argument("--max_tokens",default=2048, type=int, required=True,help="the max tokens for the judge model.")
-    parser.add_argument("--temperature",default=0.0, type=float, required=True,help="the temperature for the judge model.")
-    parser.add_argument("--top_p",default=1.0, type=float, required=True,help="the top p for the judge model.")
-    parser.add_argument("--tensor_parallel_size",default=1, type=int, required=True,help="the tensor parallel size for the judge model.")
-    args = parser.parse_args()
-    return args
-
-def pil_to_image_dict(pil_img):
-    """
-    Convert a PIL Image to an image dict with 'bytes' and 'path' keys.
-    """
-    buf = io.BytesIO()
-    pil_img.save(buf, format="PNG")  # or "JPEG" if you prefer
-    img_bytes = buf.getvalue()
-    return {'bytes': img_bytes, 'path': None}
-
-def load_aya_vision_bench_data(language):
-    mapping_language_to_dataset_name = {
-        "cs": "ces_Latn",
-        "eng": "eng_Latn",
-        "fr": "fra_Latn",
-        "de": "deu_Latn",
-        "es": "spa_Latn",
-        "it": "ita_Latn",
-        "ko": "kor_Hang",
-        "nl": "nld_Latn",
-        "pt": "por_Latn",
-        "ru": "rus_Cyrl",
-        "zh": "zho_Hans",
-    }
-    dataset = load_dataset("CohereForAI/AyaVisionBench",name=mapping_language_to_dataset_name[language],split="test")
-    logger.info(f"Loaded Aya Vision Bench dataset for language: {language}")
-    # dataset = dataset.select(range(5))
-    # Extract questions and images from the dataset
-    questions = [item["question"] for item in dataset]
-
-    images_bytes = []
-    questions = []
-    for item in dataset:
-        questions.append(item["question"])
-        pil_img = item["image"][0]
-        images_bytes.append(pil_to_image_dict(pil_img))
-    return dataset,questions,images_bytes
-
-def load_model_outputs(model_outputs_path):
-    filtered_responses = []
-    with open(model_outputs_path, 'r', encoding='utf-8') as f:
-        for line in f:
-            try:
-                data = json.loads(line.strip())
-                if 'filtered_resps' in data:
-                    # Handle both single string and list of strings cases
-                    resps = data['filtered_resps']
-                    if isinstance(resps, list):
-                        filtered_responses.extend(resps)
-                    else:
-                        filtered_responses.append(resps)
-            except json.JSONDecodeError:
-                print(f"Warning: Skipping invalid JSON line")
-                continue
-    return filtered_responses
-
-def extract_judge_message(response):
-    """
-    Extracts the output message content from a judge response object.
-    Assumes response.choices[0].message.content exists.
-    """
-    try:
-        return response.choices[0].message.content
-    except (AttributeError, IndexError, KeyError, TypeError):
-        return None
-
-if __name__ == "__main__":
-    
-    args = load_args()
-
-    # 1. Load Aya Vision Bench data
-    aya_dataset,questions,images_bytes = load_aya_vision_bench_data(args.lp)
-
-    # 2. Load model outputs
-    baseline_model_outputs = load_model_outputs(args.baseline_output_path)
-    model_outputs = load_model_outputs(args.model_output_path)
-    logger.info(f"Loaded baseline and model outputs for language: {args.lp}")
-    
-    # 3. Prepare judge_config dictionary
-    judge_config = {
-        "api_type": args.api_type,
-        "judge_model_name": args.judge_model_name,
-        "judge_prompt_type": args.judge_prompt_type,
-        "text_only": args.text_only,
-        # "api_url": os.environ.get("API_URL", None),  # or set as needed
-        "max_tokens": args.max_tokens,
-        "temperature": args.temperature,
-        "top_p": args.top_p,
-        "tensor_parallel_size": args.tensor_parallel_size,
-        # Add any other config keys your run_judge expects
-    }
-
-    # 4. Run judge
-    logger.info(f"Running judge for language: {args.lp}")
-    full_responses = run_judge(questions, model_outputs, judge_config, baseline_model_outputs, images_bytes)
-    f_resp = [extract_judge_message(resp) for resp in full_responses]
-
-    logger.info(f"Judge completed for language: {args.lp}")
-    parsed_responses = parse_judge_responses(full_responses, judge_config)
-    logger.info(f"Parsed responses completed for language: {args.lp}")
-    
-    # 7. Compute results
-    results = compute_results(parsed_responses, judge_config)
-    breakpoint()
-    # 5. Save results
-    if args.save_judge_parsed_outputs:
-        with open(os.path.join(args.output_dir, "judge_results_parsed.json"), "w") as f:
-            logger.info(f"Saving parsed responses...")
-            json.dump(parsed_responses, f)
-
-    if args.save_judge_full_responses:
-        with open(os.path.join(args.output_dir, "judge_results_full.json"), "w") as f:
-            logger.info(f"Saving full responses...")
-            json.dump(f_resp, f)
-
-    if args.save_judge_results:
-        with open(os.path.join(args.output_dir, "judge_results.json"), "w") as f:
-            logger.info(f"Saving results...")
-            json.dump(results, f)
-
-    
-    
\ No newline at end of file
diff --git a/lmms_eval/tasks/ayavisionbench/judge_templates.py b/lmms_eval/tasks/ayavisionbench/judge_templates.py
deleted file mode 100755
index 6a635ed65..000000000
--- a/lmms_eval/tasks/ayavisionbench/judge_templates.py
+++ /dev/null
@@ -1,39 +0,0 @@
-COMPARATIVE_GEN_SYSTEM_PROMPT = """\
-Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user prompt displayed below. You will be given assistant A's answer and assistant B's answer. Your job is to evaluate which assistant's answer is better, by taking into account the given image.
-
-Begin your evaluation by generating your own answer to the prompt. You must provide your answers before judging any answers. To provide your answer, take into account the given image.
-
-When evaluating the assistants' answers, compare both assistants' answers with your answer. You must identify and correct any mistakes or inaccurate information.
-
-Then consider if the assistant's answers are helpful, relevant, and concise. Helpful means the answer correctly responds to the prompt or follows the instructions. Note when user prompt has any ambiguity or more than one interpretation, it is more helpful and appropriate to ask for clarifications or more information from the user than providing an answer based on assumptions. Relevant means all parts of the response closely connect or are appropriate to what is being asked. Concise means the response is clear and not verbose or excessive.
-
-Then consider the creativity and novelty of the assistant's answers when needed. Finally, identify any missing important information in the assistants' answers that would be beneficial to include when responding to the user prompt.
-
-After providing your explanation, you must output only one of the following choices as your final verdict with a label:
-
-1. Assistant A is significantly better: [[A>>B]]
-2. Assistant A is slightly better: [[A>B]]
-3. Tie, relatively the same: [[A=B]]
-4. Assistant B is slightly better: [[B>A]]
-5. Assistant B is significantly better: [[B>>A]]
-
-Example output: "My final verdict is tie: [[A=B]]".\
-"""
-
-COMPARATIVE_GEN_USER_PROMPT = """Question:{question}\n\n<|The Start of Assistant A's Answer|>\n{answer_1}\n<|The End of Assistant A's Answer|>\n\n<|The Start of Assistant B's Answer|>\n{answer_2}\n<|The End of Assistant B's Answer|>"""
-
-DIRECT_ASSESSMENT_SYSTEM_PROMPT = """\
-You are an impartial judge and will be given an image, a question, and an answer of an AI assistant. Your job is to evaluate the quality of the answer.
-
-Begin your evaluation by generating your own answer to the prompt. You must provide your answers before judging any answers.
-
-When evaluating the assistant's answer, compare the answer with your answer. You must identify and correct any mistakes or inaccurate information.
-
-Then consider if the assistant's answer is helpful, relevant, and concise. Helpful means the answer correctly responds to the prompt or follows the instructions. Note when user prompt has any ambiguity or more than one interpretation, it is more helpful and appropriate to ask for clarifications or more information from the user than providing an answer based on assumptions. Relevant means all parts of the response closely connect or are appropriate to what is being asked. Concise means the response is clear and not verbose or excessive.
-
-After providing your explanation, you must output only a score between 0 and 100, where 0 is the worst and 100 is the best.
-
-Example output: "My final score is 85."
-"""
-
-DIRECT_ASSESSMENT_USER_PROMPT = """Question:{question}\n\n<|The Start of Assistant Answer|>\n{answer}\n<|The End of Assistant Answer|>\n\n"""
\ No newline at end of file
diff --git a/lmms_eval/tasks/ayavisionbench/judge_utils.py b/lmms_eval/tasks/ayavisionbench/judge_utils.py
deleted file mode 100644
index ea0a64a5b..000000000
--- a/lmms_eval/tasks/ayavisionbench/judge_utils.py
+++ /dev/null
@@ -1,255 +0,0 @@
-import yaml
-from pathlib import Path
-import os
-import requests
-import json
-import re
-from loguru import logger
-from dotenv import load_dotenv
-import base64
-import io
-from PIL import Image
-from tqdm import tqdm
-from typing import Dict, List, Optional, Union
-import litellm
-
-
-from lmms_eval.tasks.ayavisionbench.judge_templates import (
-    COMPARATIVE_GEN_USER_PROMPT,
-    COMPARATIVE_GEN_SYSTEM_PROMPT,
-    DIRECT_ASSESSMENT_SYSTEM_PROMPT,
-    DIRECT_ASSESSMENT_USER_PROMPT 
-)
-
-
-def img_bytes_to_url(image_dict: dict) -> str:
-    """
-    Convert image bytes to base64 URL.
-    
-    Args:
-        image_dict: Dictionary containing image bytes
-        
-    Returns:
-        str: base64 URL-encoded JPEG string
-    """
-    try:
-        # Convert bytes to PIL Image
-        image = Image.open(io.BytesIO(image_dict['bytes']))
-        
-        # Convert to RGB if needed
-        if image.mode != 'RGB':
-            image = image.convert('RGB')
-        
-        # Save as JPEG to buffer
-        buffer = io.BytesIO()
-        image.save(buffer, format="JPEG")
-        img_bytes = buffer.getvalue()
-        
-        # Convert to base64
-        base64_image = base64.b64encode(img_bytes).decode('utf-8')
-        
-        # Create data URL
-        data_url = f"data:image/jpeg;base64,{base64_image}"
-        
-        return data_url
-        
-    except Exception as e:
-        logger.error(f"Error converting image bytes to URL: {str(e)}")
-        return None
-
-def get_judge_config():
-    with open(Path(__file__).parent / "eval_judge_template.yaml", "r") as f:
-        raw_data = f.readlines()
-        safe_data = []
-        for i, line in enumerate(raw_data):
-            # remove function definition since yaml load cannot handle it
-            if "!function" not in line:
-                safe_data.append(line)
-        config = yaml.safe_load("".join(safe_data))
-    return config
-
-
-def set_prompts(judge_config,questions,preds,baseline_model_outputs=None):
-    if judge_config["judge_prompt_type"] == "comparative":
-        system_prompt = COMPARATIVE_GEN_SYSTEM_PROMPT
-        user_prompt_template = COMPARATIVE_GEN_USER_PROMPT
-        prompts = [user_prompt_template.format(question=question,answer_1=base_output,answer_2=pred) for question,pred,base_output in zip(questions,preds,baseline_model_outputs)]
-    elif judge_config["judge_prompt_type"] == "direct_assessment":
-        system_prompt = DIRECT_ASSESSMENT_SYSTEM_PROMPT
-        user_prompt_template = DIRECT_ASSESSMENT_USER_PROMPT
-        prompts = [user_prompt_template.format(question=question,answer=pred) for question,pred in zip(questions,preds)]
-    else:
-        raise ValueError(f"Invalid judge prompt type: {judge_config['judge_prompt_type']}")
-    return system_prompt, prompts
-
-
-
-def run_judge(
-    questions: List[str],
-    preds: List[str],
-    judge_config: Dict,
-    baseline_model_outputs: Optional[List[str]] = None,
-    images: Optional[Dict[str, Union[bytes, None]]] = None,
-) -> List[str]:
-    """
-    Run judge evaluation on predictions using LiteLLM.
-    
-    Args:
-        questions: List of questions to evaluate
-        preds: List of predicted outputs to evaluate
-        judge_config: Configuration dictionary for the judge
-        baseline_model_outputs: Optional list of baseline outputs to compare against
-        images: Optional dictionary containing image data with format {'bytes': bytes, 'path': None}
-        
-    Returns:
-        List[Dict]: List of parsed responses from the judge
-    """
-    logger.info(f"Selected judge type: {judge_config['judge_prompt_type']}")
-    load_dotenv()
-    
-    # Set up model configuration
-    model = judge_config["judge_model_name"]
-    
-
-
-    if judge_config["api_type"] == "anthropic":
-        if os.getenv("ANTHROPIC_API_KEY"):
-            api_key = os.getenv("ANTHROPIC_API_KEY")
-        else:
-            raise ValueError("No API key found. Please set ANTHROPIC_API_KEY environment variable or define the api_key in the judge_config")
-    elif judge_config["api_type"] == "openai":
-        if os.getenv("OPENAI_API_KEY"):
-            api_key = os.getenv("OPENAI_API_KEY")
-        else:
-            raise ValueError("No API key found. Please set OPENAI_API_KEY environment variable or define the api_key in the judge_config")
-    elif judge_config["api_type"] == "litellm":
-        if os.getenv("LITELLM_API_KEY"):
-            api_key = os.getenv("LITELLM_API_KEY")
-        else:
-            raise ValueError("No API key found. Please set LITELLM_API_KEY environment variable or define the api_key in the judge_config")
-    else:
-        raise ValueError("No API key found. Please set LITELLM_API_KEY, ANTHROPIC_API_KEY, or OPENAI_API_KEY environment variable or define the api_key in the judge_config")
-
-    
-
-    # if judge_config["text_only"] == False:
-    #     assert litellm.supports_vision(model=model),f"Selected judge model:{model} does not support vision"
-
-    # Get prompts
-    system_prompt, user_prompts = set_prompts(judge_config, questions, preds, baseline_model_outputs)
-    
-    responses = []
-    total_items = len(user_prompts)
-    logger.info(f"Running judge evaluation for {total_items} items...")
-    
-    with tqdm(total=total_items, 
-              desc="Running judge evaluation", 
-              bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]',
-              ncols=100) as pbar:
-              
-        for idx, (image, prompt) in enumerate(zip(images, user_prompts)):
-            try:
-                # Prepare messages based on whether image is present
-                if image is not None or judge_config["text_only"] == False:
-                    messages = [
-                        {"role": "system", "content": [{"type": "text", "text": system_prompt}]},
-                        {"role": "user", "content": [
-                            {"type": "text", "text": prompt},
-                            {"type": "image_url", "image_url": {"url": img_bytes_to_url(image)}}
-                        ]}
-                    ]
-                else:
-                    messages = [
-                        {"role": "system", "content": system_prompt},
-                        {"role": "user", "content": prompt}
-                    ]
-
-                # Make the LiteLLM call
-                response = litellm.completion(
-                    model=model,
-                    messages=messages,
-                    max_tokens= judge_config["max_tokens"] if "max_tokens" in judge_config else None,
-                    temperature=judge_config["temperature"] if "temperature" in judge_config else None,
-                    top_p=judge_config["top_p"] if "top_p" in judge_config else None,
-                    api_key=api_key,
-                    base_url=judge_config.get("api_url")
-                )
-                responses.append(response)
-                
-            except Exception as e:
-                logger.error(f"Error in LiteLLM call: {str(e)}")
-                responses.append(None)
-            
-            pbar.set_postfix({"Current": f"{idx+1}/{total_items}"})
-            pbar.update(1)
-    return responses
-
-def parse_comparative_response(response: str) -> str:
-    """
-    Extract verdict pattern from judge response.
-    Matches: [[A>>B]], [[A>B]], [[A=B]], [[B>A]], [[B>>A]]
-    
-    Args:
-        response: String containing judge's response
-        
-    Returns:
-        str: Matched verdict pattern or None if no match found
-    """
-    # Regex pattern to match all possible verdicts
-    pattern = r'\[\[(A>>B|A>B|A=B|B>A|B>>A)\]\]'
-    # Search for pattern in response
-    match = re.search(pattern, response)
-    # Return matched pattern or None
-    return match.group(0) if match else None
-
-
-def parse_judge_responses(responses,judge_config):
-    logger.info(f"Parsing responses for {judge_config['judge_prompt_type']} judge...")
-    parsed_responses = []
-    for response in responses:
-        if response is None:
-            parsed_responses.append(None)
-            continue
-        response_data = response["choices"][0]["message"]["content"]
-        if judge_config["judge_prompt_type"] == "comparative":
-            parsed_responses.append(parse_comparative_response(response_data))
-        elif judge_config["judge_prompt_type"] == "direct_assessment":
-            raise NotImplementedError("Direct assessment parsing not implemented yet!")
-    return parsed_responses
-
-
-def compute_results(responses,judge_config):
-    logger.info(f"Computing results for {judge_config['judge_prompt_type']} judge...")
-    if judge_config["judge_prompt_type"] == "comparative":
-        a_better_than_b = 0
-        a_significantly_better_than_b = 0
-        b_better_than_a = 0
-        b_significantly_better_than_a = 0
-        a_equal_to_b = 0
-        no_answer = 0
-        for response in responses:
-            if response == "[[A>>B]]":
-                a_significantly_better_than_b += 1  
-            elif response == "[[A>B]]":
-                a_better_than_b += 1
-            elif response == "[[B>>A]]":
-                b_significantly_better_than_a += 1
-            elif response == "[[B>A]]":
-                b_better_than_a += 1
-            elif response == "[[A=B]]":
-                a_equal_to_b += 1
-            else:
-                no_answer += 1
-        results = {
-            "baseline_better_than_model": a_better_than_b/len(responses),
-            "baseline_significantly_better_than_model": a_significantly_better_than_b/len(responses),
-            "model_better_than_baseline": b_better_than_a/len(responses),
-            "model_significantly_better_than_baseline": b_significantly_better_than_a/len(responses),
-            "model_equal_to_baseline": a_equal_to_b/len(responses),
-            "no_answer_matched": no_answer/len(responses)
-        }
-    elif judge_config["judge_prompt_type"] == "direct_assessment":
-        raise NotImplementedError("Direct assessment results computation not implemented yet!")
-    else:
-        raise ValueError(f"Invalid judge prompt type: {judge_config['judge_prompt_type']}")
-    return results
\ No newline at end of file
diff --git a/lmms_eval/tasks/ayavisionbench/utils.py b/lmms_eval/tasks/ayavisionbench/utils.py
deleted file mode 100644
index 060adbd0d..000000000
--- a/lmms_eval/tasks/ayavisionbench/utils.py
+++ /dev/null
@@ -1,131 +0,0 @@
-from PIL import Image
-from io import BytesIO
-import numpy as np
-import json
-from loguru import logger
-from pathlib import Path
-import io
-from copy import deepcopy
-
-from lmms_eval.tasks.ayavisionbench.judge_utils import (
-    get_judge_config,
-    run_judge,
-    compute_results
-)
-
-def pil_to_image_dict(pil_img):
-    """
-    Convert a PIL Image to an image dict with 'bytes' and 'path' keys.
-    """
-    buf = io.BytesIO()
-    pil_img.save(buf, format="PNG")  # or "JPEG" if you prefer
-    img_bytes = buf.getvalue()
-    return {'bytes': img_bytes, 'path': None}
-
-
-
-def load_baseline_outputs(baseline_model_outputs_path):
-    filtered_responses = []
-    with open(baseline_model_outputs_path, 'r', encoding='utf-8') as f:
-        for line in f:
-            try:
-                data = json.loads(line.strip())
-                if 'filtered_resps' in data:
-                    # Handle both single string and list of strings cases
-                    resps = data['filtered_resps']
-                    if isinstance(resps, list):
-                        filtered_responses.extend(resps)
-                    else:
-                        filtered_responses.append(resps)
-            except json.JSONDecodeError:
-                print(f"Warning: Skipping invalid JSON line")
-                continue
-    return filtered_responses
-
-
-def process_docs(docs):
-    """
-    Process documents...
-    """
-    # logger.info(f"processing docs")
-    # docs = docs.select(range(5)) # filter out some samples!
-    def copy_image_fn(example):
-        example['copy_image'] = example['image']
-        return example
-    
-    docs = docs.map(copy_image_fn)
-    return docs
-
-
-def gen_doc_to_visual(doc):
-    image = doc['image'][0]
-    image = image.convert('RGB')
-    return [image]
-
-
-def gen_doc_to_text(doc,lmms_eval_specific_kwargs=None ):
-    question = doc["question"]
-    if 'pre_prompt' in lmms_eval_specific_kwargs:
-        pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
-    else:
-        pre_prompt = ""
-    if 'post_prompt' in lmms_eval_specific_kwargs:
-        post_prompt = lmms_eval_specific_kwargs["post_prompt"]
-    else:
-        post_prompt = ""
-    return f"{pre_prompt}Question: {question}\n{post_prompt}"
-
-
-def gen_process_results(doc, results):
-    generated_texts = results[0]
-    myimg = doc['copy_image'][0]['bytes']
-    pil_img = Image.open(io.BytesIO(myimg))
-    return {"results": {
-        "id": doc["index"],
-        "image": pil_img,
-        "question": doc["question"],
-        "image_category": doc["image_source_category"],
-        "prediction": generated_texts
-        }
-    }
-
-# def aggregate_results(results):
-#     # preds = [result["prediction"] for result in results]
-#     # questions = [result["question"] for result in results]
-#     # images = [result["image"] for result in results]
-#     # # Assuming images is a list of PIL Images:
-#     # images_bytes = [pil_to_image_dict(img) for img in images.copy()]
-
-#     # judge_config = get_judge_config()
-
-#     # if judge_config["run_judge"]:
-#     #     logger.info("Judge is enabled, checking judge config...")
-#     #     if judge_config["judge_prompt_type"] == "comparative":
-#     #         logger.info("Judgement is set to comparative, checking baseline model outputs...")
-#     #         try:
-#     #             baseline_model_outputs = load_baseline_outputs(judge_config["baseline_model_outputs_path"])
-#     #             assert len(baseline_model_outputs) == len(preds)
-#     #             logger.info("Baseline model outputs loaded successfully.")
-#     #             judge_results = run_judge(questions,preds,judge_config,baseline_model_outputs,images_bytes)
-#     #         except Exception as e:
-#     #             logger.error(f"Failed while loading model outputs or running judge.")
-#     #             raise e
-            
-#     #     elif judge_config["judge_prompt_type"] == "direct assessment":
-#     #         preds = [result["prediction"] for result in results]
-#     #         judge_results = run_judge(questions,preds,judge_config,baseline_model_outputs=None,images=images_bytes)
-#     #     else:
-#     #         logger.error(f"Invalid judge prompt type: {judge_config['judge_prompt_type']}.Skipping judge...")
-#     #         judge_results = None
-        
-#     # else:
-#     #     judge_results = None
-#     #     logger.info("Judge is disabled, skipping judge...")
-
-#     # if judge_results is None:
-#     #     return {"judge_results": None}
-#     # else:
-#     #     results = compute_results(judge_results,judge_config)
-#     #     return results
-#     return {"judge_results": None}
-
diff --git a/lmms_eval/tasks/global_mmlu/README.md b/lmms_eval/tasks/global_mmlu/README.md
new file mode 100644
index 000000000..d15141024
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/README.md
@@ -0,0 +1,42 @@
+# Global-MMLU
+
+### Paper
+
+Title: `Global MMLU: Understanding and Addressing Cultural and Linguistic Biases in Multilingual Evaluation`
+
+Abstract: [https://arxiv.org/abs/2412.03304](https://arxiv.org/abs/2412.03304)
+
+Global-MMLU 🌍 is a multilingual evaluation set spanning 42 languages, including English. This dataset combines machine translations for MMLU questions along with professional translations and crowd-sourced post-edits. It also includes cultural sensitivity annotations for a subset of the questions (2850 questions per language) and classifies them as Culturally Sensitive (CS) 🗽 or Culturally Agnostic (CA) ⚖️. These annotations were collected as part of an open science initiative led by Cohere For AI in collaboration with many external collaborators from both industry and academia.
+
+Global-MMLU-Lite is a balanced collection of culturally sensitive and culturally agnostic MMLU tasks. It is designed for efficient evaluation of multilingual models in 15 languages (including English). Only languages with human translations and post-edits in the original [Global-MMLU](https://huggingface.co/datasets/CohereForAI/Global-MMLU) 🌍 dataset have been included in the lite version.
+
+Homepage: \
+[https://huggingface.co/datasets/CohereForAI/Global-MMLU](https://huggingface.co/datasets/CohereForAI/Global-MMLU) \
+[https://huggingface.co/datasets/CohereForAI/Global-MMLU-Lite](https://huggingface.co/datasets/CohereForAI/Global-MMLU-Lite)
+
+
+#### Groups
+
+* `global_mmlu_{lang}`: This group uses `Global-MMLU-Lite` benchmark which supports 14 languages.
+* `global_mmlu_full_{lang}`: This group uses `Global-MMLU` benchmark which supports 42 languages.
+
+#### Subgroups (support only for `full` version)
+
+* `global_mmlu_full_stem`
+* `global_mmlu_full_humanities`
+* `global_mmlu_full_social_sciences`
+* `global_mmlu_full_other`
+
+### Citation
+
+```bibtex
+@misc{singh2024globalmmluunderstandingaddressing,
+      title={Global MMLU: Understanding and Addressing Cultural and Linguistic Biases in Multilingual Evaluation},
+      author={Shivalika Singh and Angelika Romanou and Clémentine Fourrier and David I. Adelani and Jian Gang Ngui and Daniel Vila-Suero and Peerat Limkonchotiwat and Kelly Marchisio and Wei Qi Leong and Yosephine Susanto and Raymond Ng and Shayne Longpre and Wei-Yin Ko and Madeline Smith and Antoine Bosselut and Alice Oh and Andre F. T. Martins and Leshem Choshen and Daphne Ippolito and Enzo Ferrante and Marzieh Fadaee and Beyza Ermis and Sara Hooker},
+      year={2024},
+      eprint={2412.03304},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2412.03304},
+}
+```
diff --git a/lmms_eval/tasks/global_mmlu/default/ar/_ar_template_yaml b/lmms_eval/tasks/global_mmlu/default/ar/_ar_template_yaml
new file mode 100644
index 000000000..3fa8f23f8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ar/_ar_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU-Lite
+dataset_name: ar
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: default
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/ar/_global_mmlu_ar.yaml b/lmms_eval/tasks/global_mmlu/default/ar/_global_mmlu_ar.yaml
new file mode 100644
index 000000000..27f6e1a47
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ar/_global_mmlu_ar.yaml
@@ -0,0 +1,13 @@
+group: global_mmlu_ar
+task:
+  - global_mmlu_ar_business
+  - global_mmlu_ar_humanities
+  - global_mmlu_ar_medical
+  - global_mmlu_ar_other
+  - global_mmlu_ar_stem
+  - global_mmlu_ar_social_sciences
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_business.yaml b/lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_business.yaml
new file mode 100644
index 000000000..c7f47fdf8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_business.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_business
+task: global_mmlu_ar_business
diff --git a/lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_humanities.yaml b/lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_humanities.yaml
new file mode 100644
index 000000000..c35f1f6e9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_humanities.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_humanities
+task: global_mmlu_ar_humanities
diff --git a/lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_medical.yaml b/lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_medical.yaml
new file mode 100644
index 000000000..cb4054866
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_medical.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_medical
+task: global_mmlu_ar_medical
diff --git a/lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_other.yaml b/lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_other.yaml
new file mode 100644
index 000000000..1ffd9be89
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_other.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_other
+task: global_mmlu_ar_other
diff --git a/lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_social_sciences.yaml
new file mode 100644
index 000000000..037e25a81
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_social_sciences.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_social_sciences
+task: global_mmlu_ar_social_sciences
diff --git a/lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_stem.yaml b/lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_stem.yaml
new file mode 100644
index 000000000..f2ed28c71
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ar/global_mmlu_ar_stem.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_stem
+task: global_mmlu_ar_stem
diff --git a/lmms_eval/tasks/global_mmlu/default/ar/utils.py b/lmms_eval/tasks/global_mmlu/default/ar/utils.py
new file mode 100644
index 000000000..507a41bdc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ar/utils.py
@@ -0,0 +1,18 @@
+from functools import partial
+
+
+CATEGORIES = ["Business", "Humanities", "Medical", "Other", "STEM", "Social Sciences"]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["subject_category"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/default/bn/_bn_template_yaml b/lmms_eval/tasks/global_mmlu/default/bn/_bn_template_yaml
new file mode 100644
index 000000000..c9a234dbf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/bn/_bn_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU-Lite
+dataset_name: bn
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: default
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/bn/_global_mmlu_bn.yaml b/lmms_eval/tasks/global_mmlu/default/bn/_global_mmlu_bn.yaml
new file mode 100644
index 000000000..4098af1a2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/bn/_global_mmlu_bn.yaml
@@ -0,0 +1,13 @@
+group: global_mmlu_bn
+task:
+  - global_mmlu_bn_business
+  - global_mmlu_bn_humanities
+  - global_mmlu_bn_medical
+  - global_mmlu_bn_other
+  - global_mmlu_bn_stem
+  - global_mmlu_bn_social_sciences
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_business.yaml b/lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_business.yaml
new file mode 100644
index 000000000..c77589c30
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_business.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_business
+task: global_mmlu_bn_business
diff --git a/lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_humanities.yaml b/lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_humanities.yaml
new file mode 100644
index 000000000..da495c6d5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_humanities.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_humanities
+task: global_mmlu_bn_humanities
diff --git a/lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_medical.yaml b/lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_medical.yaml
new file mode 100644
index 000000000..867e5e4ec
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_medical.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_medical
+task: global_mmlu_bn_medical
diff --git a/lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_other.yaml b/lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_other.yaml
new file mode 100644
index 000000000..c44b6d755
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_other.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_other
+task: global_mmlu_bn_other
diff --git a/lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_social_sciences.yaml
new file mode 100644
index 000000000..7bbde182d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_social_sciences.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_social_sciences
+task: global_mmlu_bn_social_sciences
diff --git a/lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_stem.yaml b/lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_stem.yaml
new file mode 100644
index 000000000..433ba8b7a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/bn/global_mmlu_bn_stem.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_stem
+task: global_mmlu_bn_stem
diff --git a/lmms_eval/tasks/global_mmlu/default/bn/utils.py b/lmms_eval/tasks/global_mmlu/default/bn/utils.py
new file mode 100644
index 000000000..507a41bdc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/bn/utils.py
@@ -0,0 +1,18 @@
+from functools import partial
+
+
+CATEGORIES = ["Business", "Humanities", "Medical", "Other", "STEM", "Social Sciences"]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["subject_category"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/default/de/_de_template_yaml b/lmms_eval/tasks/global_mmlu/default/de/_de_template_yaml
new file mode 100644
index 000000000..7c17e2d0d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/de/_de_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU-Lite
+dataset_name: de
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: default
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/de/_global_mmlu_de.yaml b/lmms_eval/tasks/global_mmlu/default/de/_global_mmlu_de.yaml
new file mode 100644
index 000000000..1a54aaceb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/de/_global_mmlu_de.yaml
@@ -0,0 +1,13 @@
+group: global_mmlu_de
+task:
+  - global_mmlu_de_business
+  - global_mmlu_de_humanities
+  - global_mmlu_de_medical
+  - global_mmlu_de_other
+  - global_mmlu_de_stem
+  - global_mmlu_de_social_sciences
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_business.yaml b/lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_business.yaml
new file mode 100644
index 000000000..eba9514c1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_business.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_business
+task: global_mmlu_de_business
diff --git a/lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_humanities.yaml b/lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_humanities.yaml
new file mode 100644
index 000000000..d37de4919
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_humanities.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_humanities
+task: global_mmlu_de_humanities
diff --git a/lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_medical.yaml b/lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_medical.yaml
new file mode 100644
index 000000000..f114de463
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_medical.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_medical
+task: global_mmlu_de_medical
diff --git a/lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_other.yaml b/lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_other.yaml
new file mode 100644
index 000000000..d6089b2df
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_other.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_other
+task: global_mmlu_de_other
diff --git a/lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_social_sciences.yaml
new file mode 100644
index 000000000..853711f3a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_social_sciences.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_social_sciences
+task: global_mmlu_de_social_sciences
diff --git a/lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_stem.yaml b/lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_stem.yaml
new file mode 100644
index 000000000..ef66d3cf7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/de/global_mmlu_de_stem.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_stem
+task: global_mmlu_de_stem
diff --git a/lmms_eval/tasks/global_mmlu/default/de/utils.py b/lmms_eval/tasks/global_mmlu/default/de/utils.py
new file mode 100644
index 000000000..507a41bdc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/de/utils.py
@@ -0,0 +1,18 @@
+from functools import partial
+
+
+CATEGORIES = ["Business", "Humanities", "Medical", "Other", "STEM", "Social Sciences"]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["subject_category"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/default/en/_en_template_yaml b/lmms_eval/tasks/global_mmlu/default/en/_en_template_yaml
new file mode 100644
index 000000000..e24d79832
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/en/_en_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU-Lite
+dataset_name: en
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: default
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/en/_global_mmlu_en.yaml b/lmms_eval/tasks/global_mmlu/default/en/_global_mmlu_en.yaml
new file mode 100644
index 000000000..fc927412e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/en/_global_mmlu_en.yaml
@@ -0,0 +1,13 @@
+group: global_mmlu_en
+task:
+  - global_mmlu_en_business
+  - global_mmlu_en_humanities
+  - global_mmlu_en_medical
+  - global_mmlu_en_other
+  - global_mmlu_en_stem
+  - global_mmlu_en_social_sciences
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_business.yaml b/lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_business.yaml
new file mode 100644
index 000000000..aa3f4bc14
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_business.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_business
+task: global_mmlu_en_business
diff --git a/lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_humanities.yaml b/lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_humanities.yaml
new file mode 100644
index 000000000..c2a20e292
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_humanities.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_humanities
+task: global_mmlu_en_humanities
diff --git a/lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_medical.yaml b/lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_medical.yaml
new file mode 100644
index 000000000..ba9914592
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_medical.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_medical
+task: global_mmlu_en_medical
diff --git a/lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_other.yaml b/lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_other.yaml
new file mode 100644
index 000000000..c14d7657a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_other.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_other
+task: global_mmlu_en_other
diff --git a/lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_social_sciences.yaml
new file mode 100644
index 000000000..d576d2c15
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_social_sciences.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_social_sciences
+task: global_mmlu_en_social_sciences
diff --git a/lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_stem.yaml b/lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_stem.yaml
new file mode 100644
index 000000000..fd0179f2a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/en/global_mmlu_en_stem.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_stem
+task: global_mmlu_en_stem
diff --git a/lmms_eval/tasks/global_mmlu/default/en/utils.py b/lmms_eval/tasks/global_mmlu/default/en/utils.py
new file mode 100644
index 000000000..507a41bdc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/en/utils.py
@@ -0,0 +1,18 @@
+from functools import partial
+
+
+CATEGORIES = ["Business", "Humanities", "Medical", "Other", "STEM", "Social Sciences"]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["subject_category"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/default/es/_es_template_yaml b/lmms_eval/tasks/global_mmlu/default/es/_es_template_yaml
new file mode 100644
index 000000000..b0942331b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/es/_es_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU-Lite
+dataset_name: es
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: default
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/es/_global_mmlu_es.yaml b/lmms_eval/tasks/global_mmlu/default/es/_global_mmlu_es.yaml
new file mode 100644
index 000000000..614b1b0fa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/es/_global_mmlu_es.yaml
@@ -0,0 +1,13 @@
+group: global_mmlu_es
+task:
+  - global_mmlu_es_business
+  - global_mmlu_es_humanities
+  - global_mmlu_es_medical
+  - global_mmlu_es_other
+  - global_mmlu_es_stem
+  - global_mmlu_es_social_sciences
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_business.yaml b/lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_business.yaml
new file mode 100644
index 000000000..388251a2f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_business.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_business
+task: global_mmlu_es_business
diff --git a/lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_humanities.yaml b/lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_humanities.yaml
new file mode 100644
index 000000000..fd51574b0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_humanities.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_humanities
+task: global_mmlu_es_humanities
diff --git a/lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_medical.yaml b/lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_medical.yaml
new file mode 100644
index 000000000..649ad70de
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_medical.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_medical
+task: global_mmlu_es_medical
diff --git a/lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_other.yaml b/lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_other.yaml
new file mode 100644
index 000000000..878251d10
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_other.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_other
+task: global_mmlu_es_other
diff --git a/lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_social_sciences.yaml
new file mode 100644
index 000000000..1e97c6adf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_social_sciences.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_social_sciences
+task: global_mmlu_es_social_sciences
diff --git a/lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_stem.yaml b/lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_stem.yaml
new file mode 100644
index 000000000..45b4fa4af
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/es/global_mmlu_es_stem.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_stem
+task: global_mmlu_es_stem
diff --git a/lmms_eval/tasks/global_mmlu/default/es/utils.py b/lmms_eval/tasks/global_mmlu/default/es/utils.py
new file mode 100644
index 000000000..507a41bdc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/es/utils.py
@@ -0,0 +1,18 @@
+from functools import partial
+
+
+CATEGORIES = ["Business", "Humanities", "Medical", "Other", "STEM", "Social Sciences"]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["subject_category"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/default/fr/_fr_template_yaml b/lmms_eval/tasks/global_mmlu/default/fr/_fr_template_yaml
new file mode 100644
index 000000000..a2c6fc88e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/fr/_fr_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU-Lite
+dataset_name: fr
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: default
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/fr/_global_mmlu_fr.yaml b/lmms_eval/tasks/global_mmlu/default/fr/_global_mmlu_fr.yaml
new file mode 100644
index 000000000..d65a2e253
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/fr/_global_mmlu_fr.yaml
@@ -0,0 +1,13 @@
+group: global_mmlu_fr
+task:
+  - global_mmlu_fr_business
+  - global_mmlu_fr_humanities
+  - global_mmlu_fr_medical
+  - global_mmlu_fr_other
+  - global_mmlu_fr_stem
+  - global_mmlu_fr_social_sciences
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_business.yaml b/lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_business.yaml
new file mode 100644
index 000000000..49f8543b8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_business.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_business
+task: global_mmlu_fr_business
diff --git a/lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_humanities.yaml b/lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_humanities.yaml
new file mode 100644
index 000000000..35d0086b4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_humanities.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_humanities
+task: global_mmlu_fr_humanities
diff --git a/lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_medical.yaml b/lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_medical.yaml
new file mode 100644
index 000000000..e411a3475
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_medical.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_medical
+task: global_mmlu_fr_medical
diff --git a/lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_other.yaml b/lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_other.yaml
new file mode 100644
index 000000000..5bece3031
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_other.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_other
+task: global_mmlu_fr_other
diff --git a/lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_social_sciences.yaml
new file mode 100644
index 000000000..4e26ceab7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_social_sciences.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_social_sciences
+task: global_mmlu_fr_social_sciences
diff --git a/lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_stem.yaml b/lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_stem.yaml
new file mode 100644
index 000000000..6d3d1538f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/fr/global_mmlu_fr_stem.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_stem
+task: global_mmlu_fr_stem
diff --git a/lmms_eval/tasks/global_mmlu/default/fr/utils.py b/lmms_eval/tasks/global_mmlu/default/fr/utils.py
new file mode 100644
index 000000000..507a41bdc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/fr/utils.py
@@ -0,0 +1,18 @@
+from functools import partial
+
+
+CATEGORIES = ["Business", "Humanities", "Medical", "Other", "STEM", "Social Sciences"]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["subject_category"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/default/hi/_global_mmlu_hi.yaml b/lmms_eval/tasks/global_mmlu/default/hi/_global_mmlu_hi.yaml
new file mode 100644
index 000000000..406b27a67
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/hi/_global_mmlu_hi.yaml
@@ -0,0 +1,13 @@
+group: global_mmlu_hi
+task:
+  - global_mmlu_hi_business
+  - global_mmlu_hi_humanities
+  - global_mmlu_hi_medical
+  - global_mmlu_hi_other
+  - global_mmlu_hi_stem
+  - global_mmlu_hi_social_sciences
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/hi/_hi_template_yaml b/lmms_eval/tasks/global_mmlu/default/hi/_hi_template_yaml
new file mode 100644
index 000000000..180dee963
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/hi/_hi_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU-Lite
+dataset_name: hi
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: default
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_business.yaml b/lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_business.yaml
new file mode 100644
index 000000000..63b516c56
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_business.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_business
+task: global_mmlu_hi_business
diff --git a/lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_humanities.yaml b/lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_humanities.yaml
new file mode 100644
index 000000000..d8e888cd2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_humanities.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_humanities
+task: global_mmlu_hi_humanities
diff --git a/lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_medical.yaml b/lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_medical.yaml
new file mode 100644
index 000000000..46a219577
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_medical.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_medical
+task: global_mmlu_hi_medical
diff --git a/lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_other.yaml b/lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_other.yaml
new file mode 100644
index 000000000..ea242d7a3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_other.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_other
+task: global_mmlu_hi_other
diff --git a/lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_social_sciences.yaml
new file mode 100644
index 000000000..df95b8c46
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_social_sciences.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_social_sciences
+task: global_mmlu_hi_social_sciences
diff --git a/lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_stem.yaml b/lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_stem.yaml
new file mode 100644
index 000000000..acab4f12e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_stem.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_stem
+task: global_mmlu_hi_stem
diff --git a/lmms_eval/tasks/global_mmlu/default/hi/utils.py b/lmms_eval/tasks/global_mmlu/default/hi/utils.py
new file mode 100644
index 000000000..507a41bdc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/hi/utils.py
@@ -0,0 +1,18 @@
+from functools import partial
+
+
+CATEGORIES = ["Business", "Humanities", "Medical", "Other", "STEM", "Social Sciences"]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["subject_category"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/default/id/_global_mmlu_id.yaml b/lmms_eval/tasks/global_mmlu/default/id/_global_mmlu_id.yaml
new file mode 100644
index 000000000..cfe87f590
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/id/_global_mmlu_id.yaml
@@ -0,0 +1,13 @@
+group: global_mmlu_id
+task:
+  - global_mmlu_id_business
+  - global_mmlu_id_humanities
+  - global_mmlu_id_medical
+  - global_mmlu_id_other
+  - global_mmlu_id_stem
+  - global_mmlu_id_social_sciences
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/id/_id_template_yaml b/lmms_eval/tasks/global_mmlu/default/id/_id_template_yaml
new file mode 100644
index 000000000..fae80c32f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/id/_id_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU-Lite
+dataset_name: id
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: default
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_business.yaml b/lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_business.yaml
new file mode 100644
index 000000000..d8f7c1cf1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_business.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_business
+task: global_mmlu_id_business
diff --git a/lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_humanities.yaml b/lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_humanities.yaml
new file mode 100644
index 000000000..459442d44
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_humanities.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_humanities
+task: global_mmlu_id_humanities
diff --git a/lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_medical.yaml b/lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_medical.yaml
new file mode 100644
index 000000000..1fe61f138
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_medical.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_medical
+task: global_mmlu_id_medical
diff --git a/lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_other.yaml b/lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_other.yaml
new file mode 100644
index 000000000..dfdf7dd24
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_other.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_other
+task: global_mmlu_id_other
diff --git a/lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_social_sciences.yaml
new file mode 100644
index 000000000..8ac1ddf46
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_social_sciences.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_social_sciences
+task: global_mmlu_id_social_sciences
diff --git a/lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_stem.yaml b/lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_stem.yaml
new file mode 100644
index 000000000..a2230d33f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/id/global_mmlu_id_stem.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_stem
+task: global_mmlu_id_stem
diff --git a/lmms_eval/tasks/global_mmlu/default/id/utils.py b/lmms_eval/tasks/global_mmlu/default/id/utils.py
new file mode 100644
index 000000000..507a41bdc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/id/utils.py
@@ -0,0 +1,18 @@
+from functools import partial
+
+
+CATEGORIES = ["Business", "Humanities", "Medical", "Other", "STEM", "Social Sciences"]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["subject_category"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/default/it/_global_mmlu_it.yaml b/lmms_eval/tasks/global_mmlu/default/it/_global_mmlu_it.yaml
new file mode 100644
index 000000000..1378b765e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/it/_global_mmlu_it.yaml
@@ -0,0 +1,13 @@
+group: global_mmlu_it
+task:
+  - global_mmlu_it_business
+  - global_mmlu_it_humanities
+  - global_mmlu_it_medical
+  - global_mmlu_it_other
+  - global_mmlu_it_stem
+  - global_mmlu_it_social_sciences
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/it/_it_template_yaml b/lmms_eval/tasks/global_mmlu/default/it/_it_template_yaml
new file mode 100644
index 000000000..e6b1f56de
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/it/_it_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU-Lite
+dataset_name: it
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: default
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_business.yaml b/lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_business.yaml
new file mode 100644
index 000000000..dabac0a9a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_business.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_business
+task: global_mmlu_it_business
diff --git a/lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_humanities.yaml b/lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_humanities.yaml
new file mode 100644
index 000000000..6d2c923f4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_humanities.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_humanities
+task: global_mmlu_it_humanities
diff --git a/lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_medical.yaml b/lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_medical.yaml
new file mode 100644
index 000000000..25d4efc5f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_medical.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_medical
+task: global_mmlu_it_medical
diff --git a/lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_other.yaml b/lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_other.yaml
new file mode 100644
index 000000000..3e35260db
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_other.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_other
+task: global_mmlu_it_other
diff --git a/lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_social_sciences.yaml
new file mode 100644
index 000000000..bee798350
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_social_sciences.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_social_sciences
+task: global_mmlu_it_social_sciences
diff --git a/lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_stem.yaml b/lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_stem.yaml
new file mode 100644
index 000000000..04502ceff
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/it/global_mmlu_it_stem.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_stem
+task: global_mmlu_it_stem
diff --git a/lmms_eval/tasks/global_mmlu/default/it/utils.py b/lmms_eval/tasks/global_mmlu/default/it/utils.py
new file mode 100644
index 000000000..507a41bdc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/it/utils.py
@@ -0,0 +1,18 @@
+from functools import partial
+
+
+CATEGORIES = ["Business", "Humanities", "Medical", "Other", "STEM", "Social Sciences"]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["subject_category"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/default/ja/_global_mmlu_ja.yaml b/lmms_eval/tasks/global_mmlu/default/ja/_global_mmlu_ja.yaml
new file mode 100644
index 000000000..098f3b571
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ja/_global_mmlu_ja.yaml
@@ -0,0 +1,13 @@
+group: global_mmlu_ja
+task:
+  - global_mmlu_ja_business
+  - global_mmlu_ja_humanities
+  - global_mmlu_ja_medical
+  - global_mmlu_ja_other
+  - global_mmlu_ja_stem
+  - global_mmlu_ja_social_sciences
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/ja/_ja_template_yaml b/lmms_eval/tasks/global_mmlu/default/ja/_ja_template_yaml
new file mode 100644
index 000000000..5f0e4cc6c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ja/_ja_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU-Lite
+dataset_name: ja
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: default
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_business.yaml b/lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_business.yaml
new file mode 100644
index 000000000..19a5050a6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_business.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_business
+task: global_mmlu_ja_business
diff --git a/lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_humanities.yaml b/lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_humanities.yaml
new file mode 100644
index 000000000..b2d83886c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_humanities.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_humanities
+task: global_mmlu_ja_humanities
diff --git a/lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_medical.yaml b/lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_medical.yaml
new file mode 100644
index 000000000..8c0695ef0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_medical.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_medical
+task: global_mmlu_ja_medical
diff --git a/lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_other.yaml b/lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_other.yaml
new file mode 100644
index 000000000..5e72d4c0e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_other.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_other
+task: global_mmlu_ja_other
diff --git a/lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_social_sciences.yaml
new file mode 100644
index 000000000..acdabd533
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_social_sciences.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_social_sciences
+task: global_mmlu_ja_social_sciences
diff --git a/lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_stem.yaml b/lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_stem.yaml
new file mode 100644
index 000000000..b9ab07cbb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_stem.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_stem
+task: global_mmlu_ja_stem
diff --git a/lmms_eval/tasks/global_mmlu/default/ja/utils.py b/lmms_eval/tasks/global_mmlu/default/ja/utils.py
new file mode 100644
index 000000000..507a41bdc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ja/utils.py
@@ -0,0 +1,18 @@
+from functools import partial
+
+
+CATEGORIES = ["Business", "Humanities", "Medical", "Other", "STEM", "Social Sciences"]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["subject_category"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/default/ko/_global_mmlu_ko.yaml b/lmms_eval/tasks/global_mmlu/default/ko/_global_mmlu_ko.yaml
new file mode 100644
index 000000000..19f4f961e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ko/_global_mmlu_ko.yaml
@@ -0,0 +1,13 @@
+group: global_mmlu_ko
+task:
+  - global_mmlu_ko_business
+  - global_mmlu_ko_humanities
+  - global_mmlu_ko_medical
+  - global_mmlu_ko_other
+  - global_mmlu_ko_stem
+  - global_mmlu_ko_social_sciences
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/ko/_ko_template_yaml b/lmms_eval/tasks/global_mmlu/default/ko/_ko_template_yaml
new file mode 100644
index 000000000..364e159b0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ko/_ko_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU-Lite
+dataset_name: ko
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: default
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_business.yaml b/lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_business.yaml
new file mode 100644
index 000000000..2f1ce375c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_business.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_business
+task: global_mmlu_ko_business
diff --git a/lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_humanities.yaml b/lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_humanities.yaml
new file mode 100644
index 000000000..a613ff555
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_humanities.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_humanities
+task: global_mmlu_ko_humanities
diff --git a/lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_medical.yaml b/lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_medical.yaml
new file mode 100644
index 000000000..7e8710384
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_medical.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_medical
+task: global_mmlu_ko_medical
diff --git a/lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_other.yaml b/lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_other.yaml
new file mode 100644
index 000000000..3fa1c608d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_other.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_other
+task: global_mmlu_ko_other
diff --git a/lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_social_sciences.yaml
new file mode 100644
index 000000000..ad5874f9d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_social_sciences.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_social_sciences
+task: global_mmlu_ko_social_sciences
diff --git a/lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_stem.yaml b/lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_stem.yaml
new file mode 100644
index 000000000..f6c7e8ec1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_stem.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_stem
+task: global_mmlu_ko_stem
diff --git a/lmms_eval/tasks/global_mmlu/default/ko/utils.py b/lmms_eval/tasks/global_mmlu/default/ko/utils.py
new file mode 100644
index 000000000..507a41bdc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/ko/utils.py
@@ -0,0 +1,18 @@
+from functools import partial
+
+
+CATEGORIES = ["Business", "Humanities", "Medical", "Other", "STEM", "Social Sciences"]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["subject_category"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/default/pt/_global_mmlu_pt.yaml b/lmms_eval/tasks/global_mmlu/default/pt/_global_mmlu_pt.yaml
new file mode 100644
index 000000000..7a489c124
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/pt/_global_mmlu_pt.yaml
@@ -0,0 +1,13 @@
+group: global_mmlu_pt
+task:
+  - global_mmlu_pt_business
+  - global_mmlu_pt_humanities
+  - global_mmlu_pt_medical
+  - global_mmlu_pt_other
+  - global_mmlu_pt_stem
+  - global_mmlu_pt_social_sciences
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/pt/_pt_template_yaml b/lmms_eval/tasks/global_mmlu/default/pt/_pt_template_yaml
new file mode 100644
index 000000000..f1db66294
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/pt/_pt_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU-Lite
+dataset_name: pt
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: default
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_business.yaml b/lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_business.yaml
new file mode 100644
index 000000000..1e72b1687
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_business.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_business
+task: global_mmlu_pt_business
diff --git a/lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_humanities.yaml b/lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_humanities.yaml
new file mode 100644
index 000000000..7244f2a75
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_humanities.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_humanities
+task: global_mmlu_pt_humanities
diff --git a/lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_medical.yaml b/lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_medical.yaml
new file mode 100644
index 000000000..44776f2cf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_medical.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_medical
+task: global_mmlu_pt_medical
diff --git a/lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_other.yaml b/lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_other.yaml
new file mode 100644
index 000000000..b61212018
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_other.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_other
+task: global_mmlu_pt_other
diff --git a/lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_social_sciences.yaml
new file mode 100644
index 000000000..949d346ec
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_social_sciences.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_social_sciences
+task: global_mmlu_pt_social_sciences
diff --git a/lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_stem.yaml b/lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_stem.yaml
new file mode 100644
index 000000000..9f194c115
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/pt/global_mmlu_pt_stem.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_stem
+task: global_mmlu_pt_stem
diff --git a/lmms_eval/tasks/global_mmlu/default/pt/utils.py b/lmms_eval/tasks/global_mmlu/default/pt/utils.py
new file mode 100644
index 000000000..507a41bdc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/pt/utils.py
@@ -0,0 +1,18 @@
+from functools import partial
+
+
+CATEGORIES = ["Business", "Humanities", "Medical", "Other", "STEM", "Social Sciences"]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["subject_category"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/default/sw/_global_mmlu_sw.yaml b/lmms_eval/tasks/global_mmlu/default/sw/_global_mmlu_sw.yaml
new file mode 100644
index 000000000..b3913d244
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/sw/_global_mmlu_sw.yaml
@@ -0,0 +1,13 @@
+group: global_mmlu_sw
+task:
+  - global_mmlu_sw_business
+  - global_mmlu_sw_humanities
+  - global_mmlu_sw_medical
+  - global_mmlu_sw_other
+  - global_mmlu_sw_stem
+  - global_mmlu_sw_social_sciences
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/sw/_sw_template_yaml b/lmms_eval/tasks/global_mmlu/default/sw/_sw_template_yaml
new file mode 100644
index 000000000..33edff382
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/sw/_sw_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU-Lite
+dataset_name: sw
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: default
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_business.yaml b/lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_business.yaml
new file mode 100644
index 000000000..a53ca478f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_business.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_business
+task: global_mmlu_sw_business
diff --git a/lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_humanities.yaml b/lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_humanities.yaml
new file mode 100644
index 000000000..4687df760
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_humanities.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_humanities
+task: global_mmlu_sw_humanities
diff --git a/lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_medical.yaml b/lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_medical.yaml
new file mode 100644
index 000000000..76240ea3d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_medical.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_medical
+task: global_mmlu_sw_medical
diff --git a/lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_other.yaml b/lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_other.yaml
new file mode 100644
index 000000000..7c3bfda2e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_other.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_other
+task: global_mmlu_sw_other
diff --git a/lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_social_sciences.yaml
new file mode 100644
index 000000000..4a77aa2ba
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_social_sciences.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_social_sciences
+task: global_mmlu_sw_social_sciences
diff --git a/lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_stem.yaml b/lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_stem.yaml
new file mode 100644
index 000000000..d6faf18b1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/sw/global_mmlu_sw_stem.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_stem
+task: global_mmlu_sw_stem
diff --git a/lmms_eval/tasks/global_mmlu/default/sw/utils.py b/lmms_eval/tasks/global_mmlu/default/sw/utils.py
new file mode 100644
index 000000000..507a41bdc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/sw/utils.py
@@ -0,0 +1,18 @@
+from functools import partial
+
+
+CATEGORIES = ["Business", "Humanities", "Medical", "Other", "STEM", "Social Sciences"]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["subject_category"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/default/yo/_global_mmlu_yo.yaml b/lmms_eval/tasks/global_mmlu/default/yo/_global_mmlu_yo.yaml
new file mode 100644
index 000000000..14df221ad
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/yo/_global_mmlu_yo.yaml
@@ -0,0 +1,13 @@
+group: global_mmlu_yo
+task:
+  - global_mmlu_yo_business
+  - global_mmlu_yo_humanities
+  - global_mmlu_yo_medical
+  - global_mmlu_yo_other
+  - global_mmlu_yo_stem
+  - global_mmlu_yo_social_sciences
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/yo/_yo_template_yaml b/lmms_eval/tasks/global_mmlu/default/yo/_yo_template_yaml
new file mode 100644
index 000000000..6cdd0a035
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/yo/_yo_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU-Lite
+dataset_name: yo
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: default
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_business.yaml b/lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_business.yaml
new file mode 100644
index 000000000..162a97cf0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_business.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_business
+task: global_mmlu_yo_business
diff --git a/lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_humanities.yaml b/lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_humanities.yaml
new file mode 100644
index 000000000..5befbc12e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_humanities.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_humanities
+task: global_mmlu_yo_humanities
diff --git a/lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_medical.yaml b/lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_medical.yaml
new file mode 100644
index 000000000..d48d02088
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_medical.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_medical
+task: global_mmlu_yo_medical
diff --git a/lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_other.yaml b/lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_other.yaml
new file mode 100644
index 000000000..5e407c2cf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_other.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_other
+task: global_mmlu_yo_other
diff --git a/lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_social_sciences.yaml
new file mode 100644
index 000000000..c85596aa6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_social_sciences.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_social_sciences
+task: global_mmlu_yo_social_sciences
diff --git a/lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_stem.yaml b/lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_stem.yaml
new file mode 100644
index 000000000..a19e1e8dc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/yo/global_mmlu_yo_stem.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_stem
+task: global_mmlu_yo_stem
diff --git a/lmms_eval/tasks/global_mmlu/default/yo/utils.py b/lmms_eval/tasks/global_mmlu/default/yo/utils.py
new file mode 100644
index 000000000..507a41bdc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/yo/utils.py
@@ -0,0 +1,18 @@
+from functools import partial
+
+
+CATEGORIES = ["Business", "Humanities", "Medical", "Other", "STEM", "Social Sciences"]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["subject_category"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/default/zh/_global_mmlu_zh.yaml b/lmms_eval/tasks/global_mmlu/default/zh/_global_mmlu_zh.yaml
new file mode 100644
index 000000000..212a33fc9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/zh/_global_mmlu_zh.yaml
@@ -0,0 +1,13 @@
+group: global_mmlu_zh
+task:
+  - global_mmlu_zh_business
+  - global_mmlu_zh_humanities
+  - global_mmlu_zh_medical
+  - global_mmlu_zh_other
+  - global_mmlu_zh_stem
+  - global_mmlu_zh_social_sciences
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/zh/_zh_template_yaml b/lmms_eval/tasks/global_mmlu/default/zh/_zh_template_yaml
new file mode 100644
index 000000000..eeb1e7b9c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/zh/_zh_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU-Lite
+dataset_name: zh
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: default
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_business.yaml b/lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_business.yaml
new file mode 100644
index 000000000..aa0a689ae
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_business.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_business
+task: global_mmlu_zh_business
diff --git a/lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_humanities.yaml b/lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_humanities.yaml
new file mode 100644
index 000000000..823854b95
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_humanities.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_humanities
+task: global_mmlu_zh_humanities
diff --git a/lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_medical.yaml b/lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_medical.yaml
new file mode 100644
index 000000000..f1f7a7d58
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_medical.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_medical
+task: global_mmlu_zh_medical
diff --git a/lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_other.yaml b/lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_other.yaml
new file mode 100644
index 000000000..a3beae834
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_other.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_other
+task: global_mmlu_zh_other
diff --git a/lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_social_sciences.yaml
new file mode 100644
index 000000000..1891a45ae
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_social_sciences.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_social_sciences
+task: global_mmlu_zh_social_sciences
diff --git a/lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_stem.yaml b/lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_stem.yaml
new file mode 100644
index 000000000..6a9f4f05d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/zh/global_mmlu_zh_stem.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_stem
+task: global_mmlu_zh_stem
diff --git a/lmms_eval/tasks/global_mmlu/default/zh/utils.py b/lmms_eval/tasks/global_mmlu/default/zh/utils.py
new file mode 100644
index 000000000..507a41bdc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/default/zh/utils.py
@@ -0,0 +1,18 @@
+from functools import partial
+
+
+CATEGORIES = ["Business", "Humanities", "Medical", "Other", "STEM", "Social Sciences"]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["subject_category"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/am/_am_template_yaml b/lmms_eval/tasks/global_mmlu/full/am/_am_template_yaml
new file mode 100644
index 000000000..f52152bb2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/_am_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: am
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/am/_global_mmlu_full_am.yaml b/lmms_eval/tasks/global_mmlu/full/am/_global_mmlu_full_am.yaml
new file mode 100644
index 000000000..555bfd868
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/_global_mmlu_full_am.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_am
+task:
+  - global_mmlu_full_am_stem
+  - global_mmlu_full_am_other
+  - global_mmlu_full_am_social_sciences
+  - global_mmlu_full_am_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/am/_global_mmlu_full_am_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/am/_global_mmlu_full_am_humanities.yaml
new file mode 100644
index 000000000..e250d14c2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/_global_mmlu_full_am_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_am_humanities
+task:
+  - global_mmlu_full_am_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/am/_global_mmlu_full_am_other.yaml b/lmms_eval/tasks/global_mmlu/full/am/_global_mmlu_full_am_other.yaml
new file mode 100644
index 000000000..4b5151ce7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/_global_mmlu_full_am_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_am_other
+task:
+  - global_mmlu_full_am_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/am/_global_mmlu_full_am_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/am/_global_mmlu_full_am_social_sciences.yaml
new file mode 100644
index 000000000..f0fbcc1b7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/_global_mmlu_full_am_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_am_social_sciences
+task:
+  - global_mmlu_full_am_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/am/_global_mmlu_full_am_stem.yaml b/lmms_eval/tasks/global_mmlu/full/am/_global_mmlu_full_am_stem.yaml
new file mode 100644
index 000000000..b67dfdb75
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/_global_mmlu_full_am_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_am_stem
+task:
+  - global_mmlu_full_am_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_abstract_algebra.yaml
new file mode 100644
index 000000000..06a70dd84
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_am_stem_tasks
+task: global_mmlu_full_am_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_anatomy.yaml
new file mode 100644
index 000000000..7914c3b05
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_am_stem_tasks
+task: global_mmlu_full_am_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_astronomy.yaml
new file mode 100644
index 000000000..4e7e2a047
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_am_stem_tasks
+task: global_mmlu_full_am_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_business_ethics.yaml
new file mode 100644
index 000000000..a98a9597e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_am_other_tasks
+task: global_mmlu_full_am_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_clinical_knowledge.yaml
new file mode 100644
index 000000000..4c25627f6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_am_other_tasks
+task: global_mmlu_full_am_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_biology.yaml
new file mode 100644
index 000000000..a8b6661b5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_am_stem_tasks
+task: global_mmlu_full_am_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_chemistry.yaml
new file mode 100644
index 000000000..b0d2d2a84
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_am_stem_tasks
+task: global_mmlu_full_am_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_computer_science.yaml
new file mode 100644
index 000000000..b5c52a82d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_am_stem_tasks
+task: global_mmlu_full_am_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_mathematics.yaml
new file mode 100644
index 000000000..0b73422e3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_am_stem_tasks
+task: global_mmlu_full_am_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_medicine.yaml
new file mode 100644
index 000000000..bd36f40f2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_am_other_tasks
+task: global_mmlu_full_am_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_physics.yaml
new file mode 100644
index 000000000..009fdc1a7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_am_stem_tasks
+task: global_mmlu_full_am_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_computer_security.yaml
new file mode 100644
index 000000000..3df6247b9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_am_stem_tasks
+task: global_mmlu_full_am_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_conceptual_physics.yaml
new file mode 100644
index 000000000..4115ea024
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_am_stem_tasks
+task: global_mmlu_full_am_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_econometrics.yaml
new file mode 100644
index 000000000..87dd12cab
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_am_social_sciences_tasks
+task: global_mmlu_full_am_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_electrical_engineering.yaml
new file mode 100644
index 000000000..d8f726193
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_am_stem_tasks
+task: global_mmlu_full_am_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_elementary_mathematics.yaml
new file mode 100644
index 000000000..455563f1c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_am_stem_tasks
+task: global_mmlu_full_am_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_formal_logic.yaml
new file mode 100644
index 000000000..5c5babd4d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_am_humanities_tasks
+task: global_mmlu_full_am_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_global_facts.yaml
new file mode 100644
index 000000000..b59d47e45
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_am_other_tasks
+task: global_mmlu_full_am_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_biology.yaml
new file mode 100644
index 000000000..680d4ecac
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_am_stem_tasks
+task: global_mmlu_full_am_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_chemistry.yaml
new file mode 100644
index 000000000..96af7940f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_am_stem_tasks
+task: global_mmlu_full_am_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_computer_science.yaml
new file mode 100644
index 000000000..6cd19227b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_am_stem_tasks
+task: global_mmlu_full_am_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_european_history.yaml
new file mode 100644
index 000000000..e02491426
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_am_humanities_tasks
+task: global_mmlu_full_am_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_geography.yaml
new file mode 100644
index 000000000..b4925a542
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_am_social_sciences_tasks
+task: global_mmlu_full_am_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..d63f1d353
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_am_social_sciences_tasks
+task: global_mmlu_full_am_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..3c8a0ea68
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_am_social_sciences_tasks
+task: global_mmlu_full_am_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_mathematics.yaml
new file mode 100644
index 000000000..76a8c3d35
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_am_stem_tasks
+task: global_mmlu_full_am_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_microeconomics.yaml
new file mode 100644
index 000000000..1acbf4e1c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_am_social_sciences_tasks
+task: global_mmlu_full_am_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_physics.yaml
new file mode 100644
index 000000000..dcfd9bb9f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_am_stem_tasks
+task: global_mmlu_full_am_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_psychology.yaml
new file mode 100644
index 000000000..2dd64dc18
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_am_social_sciences_tasks
+task: global_mmlu_full_am_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_statistics.yaml
new file mode 100644
index 000000000..a523f443a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_am_stem_tasks
+task: global_mmlu_full_am_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_us_history.yaml
new file mode 100644
index 000000000..ce233f44f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_am_humanities_tasks
+task: global_mmlu_full_am_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_world_history.yaml
new file mode 100644
index 000000000..20aeca5e2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_am_humanities_tasks
+task: global_mmlu_full_am_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_human_aging.yaml
new file mode 100644
index 000000000..18e95e405
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_am_other_tasks
+task: global_mmlu_full_am_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_human_sexuality.yaml
new file mode 100644
index 000000000..140f23294
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_am_social_sciences_tasks
+task: global_mmlu_full_am_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_international_law.yaml
new file mode 100644
index 000000000..10a2d638d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_am_humanities_tasks
+task: global_mmlu_full_am_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_jurisprudence.yaml
new file mode 100644
index 000000000..cd9827425
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_am_humanities_tasks
+task: global_mmlu_full_am_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_logical_fallacies.yaml
new file mode 100644
index 000000000..2faf735c7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_am_humanities_tasks
+task: global_mmlu_full_am_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_machine_learning.yaml
new file mode 100644
index 000000000..7f5c8e9b8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_am_stem_tasks
+task: global_mmlu_full_am_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_management.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_management.yaml
new file mode 100644
index 000000000..08d080a8f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_am_other_tasks
+task: global_mmlu_full_am_management
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_marketing.yaml
new file mode 100644
index 000000000..52b4f7c6c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_am_other_tasks
+task: global_mmlu_full_am_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_medical_genetics.yaml
new file mode 100644
index 000000000..32bd2432b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_am_other_tasks
+task: global_mmlu_full_am_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_miscellaneous.yaml
new file mode 100644
index 000000000..ed5d610dc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_am_other_tasks
+task: global_mmlu_full_am_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_moral_disputes.yaml
new file mode 100644
index 000000000..bddaebc75
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_am_humanities_tasks
+task: global_mmlu_full_am_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_moral_scenarios.yaml
new file mode 100644
index 000000000..fda69f316
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_am_humanities_tasks
+task: global_mmlu_full_am_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_nutrition.yaml
new file mode 100644
index 000000000..bb0cb08b8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_am_other_tasks
+task: global_mmlu_full_am_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_philosophy.yaml
new file mode 100644
index 000000000..484c015ea
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_am_humanities_tasks
+task: global_mmlu_full_am_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_prehistory.yaml
new file mode 100644
index 000000000..6e104f48a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_am_humanities_tasks
+task: global_mmlu_full_am_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_professional_accounting.yaml
new file mode 100644
index 000000000..50c9fe501
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_am_other_tasks
+task: global_mmlu_full_am_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_professional_law.yaml
new file mode 100644
index 000000000..df2cf26cf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_am_humanities_tasks
+task: global_mmlu_full_am_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_professional_medicine.yaml
new file mode 100644
index 000000000..c28605288
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_am_other_tasks
+task: global_mmlu_full_am_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_professional_psychology.yaml
new file mode 100644
index 000000000..8562a28de
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_am_social_sciences_tasks
+task: global_mmlu_full_am_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_public_relations.yaml
new file mode 100644
index 000000000..5cb3186c9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_am_social_sciences_tasks
+task: global_mmlu_full_am_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_security_studies.yaml
new file mode 100644
index 000000000..6aa8575ee
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_am_social_sciences_tasks
+task: global_mmlu_full_am_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_sociology.yaml
new file mode 100644
index 000000000..60005babc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_am_social_sciences_tasks
+task: global_mmlu_full_am_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_us_foreign_policy.yaml
new file mode 100644
index 000000000..374fb14ad
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_am_social_sciences_tasks
+task: global_mmlu_full_am_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_virology.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_virology.yaml
new file mode 100644
index 000000000..9f2352992
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_am_other_tasks
+task: global_mmlu_full_am_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_world_religions.yaml
new file mode 100644
index 000000000..c169a0483
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/global_mmlu_full_am_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _am_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_am_humanities_tasks
+task: global_mmlu_full_am_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/am/utils.py b/lmms_eval/tasks/global_mmlu/full/am/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/am/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/_ar_template_yaml b/lmms_eval/tasks/global_mmlu/full/ar/_ar_template_yaml
new file mode 100644
index 000000000..768bb7f97
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/_ar_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: ar
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/_global_mmlu_full_ar.yaml b/lmms_eval/tasks/global_mmlu/full/ar/_global_mmlu_full_ar.yaml
new file mode 100644
index 000000000..83340da0d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/_global_mmlu_full_ar.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_ar
+task:
+  - global_mmlu_full_ar_stem
+  - global_mmlu_full_ar_other
+  - global_mmlu_full_ar_social_sciences
+  - global_mmlu_full_ar_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/_global_mmlu_full_ar_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/ar/_global_mmlu_full_ar_humanities.yaml
new file mode 100644
index 000000000..cfa6d80a2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/_global_mmlu_full_ar_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ar_humanities
+task:
+  - global_mmlu_full_ar_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/_global_mmlu_full_ar_other.yaml b/lmms_eval/tasks/global_mmlu/full/ar/_global_mmlu_full_ar_other.yaml
new file mode 100644
index 000000000..26603f334
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/_global_mmlu_full_ar_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ar_other
+task:
+  - global_mmlu_full_ar_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/_global_mmlu_full_ar_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/ar/_global_mmlu_full_ar_social_sciences.yaml
new file mode 100644
index 000000000..aca95bc2a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/_global_mmlu_full_ar_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ar_social_sciences
+task:
+  - global_mmlu_full_ar_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/_global_mmlu_full_ar_stem.yaml b/lmms_eval/tasks/global_mmlu/full/ar/_global_mmlu_full_ar_stem.yaml
new file mode 100644
index 000000000..b91e6c9bf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/_global_mmlu_full_ar_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ar_stem
+task:
+  - global_mmlu_full_ar_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_abstract_algebra.yaml
new file mode 100644
index 000000000..1f044b044
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_ar_stem_tasks
+task: global_mmlu_full_ar_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_anatomy.yaml
new file mode 100644
index 000000000..cd5d09636
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_ar_stem_tasks
+task: global_mmlu_full_ar_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_astronomy.yaml
new file mode 100644
index 000000000..d21c00b7e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_ar_stem_tasks
+task: global_mmlu_full_ar_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_business_ethics.yaml
new file mode 100644
index 000000000..a73f5f2d7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_ar_other_tasks
+task: global_mmlu_full_ar_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_clinical_knowledge.yaml
new file mode 100644
index 000000000..a9c3d0789
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_ar_other_tasks
+task: global_mmlu_full_ar_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_biology.yaml
new file mode 100644
index 000000000..6fba6a1b7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_ar_stem_tasks
+task: global_mmlu_full_ar_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_chemistry.yaml
new file mode 100644
index 000000000..386ba52dc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_ar_stem_tasks
+task: global_mmlu_full_ar_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_computer_science.yaml
new file mode 100644
index 000000000..9b846715c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_ar_stem_tasks
+task: global_mmlu_full_ar_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_mathematics.yaml
new file mode 100644
index 000000000..c8d8d090f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_ar_stem_tasks
+task: global_mmlu_full_ar_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_medicine.yaml
new file mode 100644
index 000000000..b988cfee2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_ar_other_tasks
+task: global_mmlu_full_ar_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_physics.yaml
new file mode 100644
index 000000000..008a39dc5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_ar_stem_tasks
+task: global_mmlu_full_ar_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_computer_security.yaml
new file mode 100644
index 000000000..34a935357
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_ar_stem_tasks
+task: global_mmlu_full_ar_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_conceptual_physics.yaml
new file mode 100644
index 000000000..ea20efa5b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_ar_stem_tasks
+task: global_mmlu_full_ar_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_econometrics.yaml
new file mode 100644
index 000000000..3a7579011
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_ar_social_sciences_tasks
+task: global_mmlu_full_ar_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_electrical_engineering.yaml
new file mode 100644
index 000000000..31a4e22ef
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_ar_stem_tasks
+task: global_mmlu_full_ar_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_elementary_mathematics.yaml
new file mode 100644
index 000000000..25f4adb9e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_ar_stem_tasks
+task: global_mmlu_full_ar_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_formal_logic.yaml
new file mode 100644
index 000000000..b2792d567
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_ar_humanities_tasks
+task: global_mmlu_full_ar_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_global_facts.yaml
new file mode 100644
index 000000000..af1bf60bb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_ar_other_tasks
+task: global_mmlu_full_ar_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_biology.yaml
new file mode 100644
index 000000000..8f7eaff72
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_ar_stem_tasks
+task: global_mmlu_full_ar_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_chemistry.yaml
new file mode 100644
index 000000000..8f56395b1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_ar_stem_tasks
+task: global_mmlu_full_ar_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_computer_science.yaml
new file mode 100644
index 000000000..6e388aed8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_ar_stem_tasks
+task: global_mmlu_full_ar_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_european_history.yaml
new file mode 100644
index 000000000..741584c5f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_ar_humanities_tasks
+task: global_mmlu_full_ar_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_geography.yaml
new file mode 100644
index 000000000..3c376967a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_ar_social_sciences_tasks
+task: global_mmlu_full_ar_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..c71ada9b6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_ar_social_sciences_tasks
+task: global_mmlu_full_ar_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..0b5f32678
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_ar_social_sciences_tasks
+task: global_mmlu_full_ar_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_mathematics.yaml
new file mode 100644
index 000000000..cb259ac24
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_ar_stem_tasks
+task: global_mmlu_full_ar_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_microeconomics.yaml
new file mode 100644
index 000000000..c4ab308b3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_ar_social_sciences_tasks
+task: global_mmlu_full_ar_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_physics.yaml
new file mode 100644
index 000000000..68180e5d6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_ar_stem_tasks
+task: global_mmlu_full_ar_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_psychology.yaml
new file mode 100644
index 000000000..e727ad091
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_ar_social_sciences_tasks
+task: global_mmlu_full_ar_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_statistics.yaml
new file mode 100644
index 000000000..8ff9dd0ba
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_ar_stem_tasks
+task: global_mmlu_full_ar_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_us_history.yaml
new file mode 100644
index 000000000..668991cfd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_ar_humanities_tasks
+task: global_mmlu_full_ar_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_world_history.yaml
new file mode 100644
index 000000000..1df9a5533
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_ar_humanities_tasks
+task: global_mmlu_full_ar_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_human_aging.yaml
new file mode 100644
index 000000000..515a40f0c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_ar_other_tasks
+task: global_mmlu_full_ar_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_human_sexuality.yaml
new file mode 100644
index 000000000..24caceac8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_ar_social_sciences_tasks
+task: global_mmlu_full_ar_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_international_law.yaml
new file mode 100644
index 000000000..a5aee4b29
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_ar_humanities_tasks
+task: global_mmlu_full_ar_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_jurisprudence.yaml
new file mode 100644
index 000000000..377812084
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_ar_humanities_tasks
+task: global_mmlu_full_ar_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_logical_fallacies.yaml
new file mode 100644
index 000000000..4365730e1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_ar_humanities_tasks
+task: global_mmlu_full_ar_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_machine_learning.yaml
new file mode 100644
index 000000000..e1fc86e2e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_ar_stem_tasks
+task: global_mmlu_full_ar_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_management.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_management.yaml
new file mode 100644
index 000000000..4dc7c8c00
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_ar_other_tasks
+task: global_mmlu_full_ar_management
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_marketing.yaml
new file mode 100644
index 000000000..371fb521c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_ar_other_tasks
+task: global_mmlu_full_ar_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_medical_genetics.yaml
new file mode 100644
index 000000000..c080b6455
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_ar_other_tasks
+task: global_mmlu_full_ar_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_miscellaneous.yaml
new file mode 100644
index 000000000..7d593ecbf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_ar_other_tasks
+task: global_mmlu_full_ar_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_moral_disputes.yaml
new file mode 100644
index 000000000..4021a93e6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_ar_humanities_tasks
+task: global_mmlu_full_ar_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_moral_scenarios.yaml
new file mode 100644
index 000000000..f09edd006
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_ar_humanities_tasks
+task: global_mmlu_full_ar_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_nutrition.yaml
new file mode 100644
index 000000000..8d8577cb6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_ar_other_tasks
+task: global_mmlu_full_ar_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_philosophy.yaml
new file mode 100644
index 000000000..733b77ce9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_ar_humanities_tasks
+task: global_mmlu_full_ar_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_prehistory.yaml
new file mode 100644
index 000000000..4d1bf1414
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_ar_humanities_tasks
+task: global_mmlu_full_ar_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_accounting.yaml
new file mode 100644
index 000000000..45b07299c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_ar_other_tasks
+task: global_mmlu_full_ar_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_law.yaml
new file mode 100644
index 000000000..6e33b5836
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_ar_humanities_tasks
+task: global_mmlu_full_ar_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_medicine.yaml
new file mode 100644
index 000000000..4cd0a17a0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_ar_other_tasks
+task: global_mmlu_full_ar_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_psychology.yaml
new file mode 100644
index 000000000..f035162d0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_ar_social_sciences_tasks
+task: global_mmlu_full_ar_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_public_relations.yaml
new file mode 100644
index 000000000..3d4dd34ff
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_ar_social_sciences_tasks
+task: global_mmlu_full_ar_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_security_studies.yaml
new file mode 100644
index 000000000..f2245b527
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_ar_social_sciences_tasks
+task: global_mmlu_full_ar_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_sociology.yaml
new file mode 100644
index 000000000..dd9203057
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_ar_social_sciences_tasks
+task: global_mmlu_full_ar_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_us_foreign_policy.yaml
new file mode 100644
index 000000000..0f38b8555
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_ar_social_sciences_tasks
+task: global_mmlu_full_ar_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_virology.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_virology.yaml
new file mode 100644
index 000000000..f3be1f8f9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_ar_other_tasks
+task: global_mmlu_full_ar_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_world_religions.yaml
new file mode 100644
index 000000000..7c7f01a50
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ar_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_ar_humanities_tasks
+task: global_mmlu_full_ar_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/ar/utils.py b/lmms_eval/tasks/global_mmlu/full/ar/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ar/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/_bn_template_yaml b/lmms_eval/tasks/global_mmlu/full/bn/_bn_template_yaml
new file mode 100644
index 000000000..f388063d0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/_bn_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: bn
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/_global_mmlu_full_bn.yaml b/lmms_eval/tasks/global_mmlu/full/bn/_global_mmlu_full_bn.yaml
new file mode 100644
index 000000000..135b4bf5d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/_global_mmlu_full_bn.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_bn
+task:
+  - global_mmlu_full_bn_stem
+  - global_mmlu_full_bn_other
+  - global_mmlu_full_bn_social_sciences
+  - global_mmlu_full_bn_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_humanities.yaml
new file mode 100644
index 000000000..acd1ab011
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_bn_humanities
+task:
+  - global_mmlu_full_bn_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_other.yaml b/lmms_eval/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_other.yaml
new file mode 100644
index 000000000..d2160298f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_bn_other
+task:
+  - global_mmlu_full_bn_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_social_sciences.yaml
new file mode 100644
index 000000000..c359b3598
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_bn_social_sciences
+task:
+  - global_mmlu_full_bn_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_stem.yaml b/lmms_eval/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_stem.yaml
new file mode 100644
index 000000000..2c78c4cea
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/_global_mmlu_full_bn_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_bn_stem
+task:
+  - global_mmlu_full_bn_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_abstract_algebra.yaml
new file mode 100644
index 000000000..5bb7bb61b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_bn_stem_tasks
+task: global_mmlu_full_bn_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_anatomy.yaml
new file mode 100644
index 000000000..d49070f11
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_bn_stem_tasks
+task: global_mmlu_full_bn_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_astronomy.yaml
new file mode 100644
index 000000000..2e6dbc971
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_bn_stem_tasks
+task: global_mmlu_full_bn_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_business_ethics.yaml
new file mode 100644
index 000000000..8c45a0e2d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_bn_other_tasks
+task: global_mmlu_full_bn_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_clinical_knowledge.yaml
new file mode 100644
index 000000000..97e17570b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_bn_other_tasks
+task: global_mmlu_full_bn_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_biology.yaml
new file mode 100644
index 000000000..9bf0b34cb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_bn_stem_tasks
+task: global_mmlu_full_bn_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_chemistry.yaml
new file mode 100644
index 000000000..cb5a26005
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_bn_stem_tasks
+task: global_mmlu_full_bn_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_computer_science.yaml
new file mode 100644
index 000000000..ecd60e54e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_bn_stem_tasks
+task: global_mmlu_full_bn_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_mathematics.yaml
new file mode 100644
index 000000000..5fb69d57f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_bn_stem_tasks
+task: global_mmlu_full_bn_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_medicine.yaml
new file mode 100644
index 000000000..442045f98
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_bn_other_tasks
+task: global_mmlu_full_bn_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_physics.yaml
new file mode 100644
index 000000000..6849ffbb2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_bn_stem_tasks
+task: global_mmlu_full_bn_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_computer_security.yaml
new file mode 100644
index 000000000..184097f8b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_bn_stem_tasks
+task: global_mmlu_full_bn_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_conceptual_physics.yaml
new file mode 100644
index 000000000..4dc8a2c20
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_bn_stem_tasks
+task: global_mmlu_full_bn_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_econometrics.yaml
new file mode 100644
index 000000000..941f63557
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_bn_social_sciences_tasks
+task: global_mmlu_full_bn_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_electrical_engineering.yaml
new file mode 100644
index 000000000..5918b08aa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_bn_stem_tasks
+task: global_mmlu_full_bn_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_elementary_mathematics.yaml
new file mode 100644
index 000000000..8f7d1f105
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_bn_stem_tasks
+task: global_mmlu_full_bn_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_formal_logic.yaml
new file mode 100644
index 000000000..b54c80db0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_bn_humanities_tasks
+task: global_mmlu_full_bn_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_global_facts.yaml
new file mode 100644
index 000000000..371d61cd6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_bn_other_tasks
+task: global_mmlu_full_bn_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_biology.yaml
new file mode 100644
index 000000000..4f2c87311
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_bn_stem_tasks
+task: global_mmlu_full_bn_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_chemistry.yaml
new file mode 100644
index 000000000..0022c8242
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_bn_stem_tasks
+task: global_mmlu_full_bn_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_computer_science.yaml
new file mode 100644
index 000000000..62ed6c6c1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_bn_stem_tasks
+task: global_mmlu_full_bn_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_european_history.yaml
new file mode 100644
index 000000000..b9118a11c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_bn_humanities_tasks
+task: global_mmlu_full_bn_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_geography.yaml
new file mode 100644
index 000000000..a7fa3c1ba
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_bn_social_sciences_tasks
+task: global_mmlu_full_bn_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..067ec0ead
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_bn_social_sciences_tasks
+task: global_mmlu_full_bn_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..12c775ab4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_bn_social_sciences_tasks
+task: global_mmlu_full_bn_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_mathematics.yaml
new file mode 100644
index 000000000..82809b152
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_bn_stem_tasks
+task: global_mmlu_full_bn_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_microeconomics.yaml
new file mode 100644
index 000000000..a14eb7031
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_bn_social_sciences_tasks
+task: global_mmlu_full_bn_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_physics.yaml
new file mode 100644
index 000000000..a84f85fd0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_bn_stem_tasks
+task: global_mmlu_full_bn_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_psychology.yaml
new file mode 100644
index 000000000..5b10c59da
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_bn_social_sciences_tasks
+task: global_mmlu_full_bn_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_statistics.yaml
new file mode 100644
index 000000000..f4231ea26
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_bn_stem_tasks
+task: global_mmlu_full_bn_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_us_history.yaml
new file mode 100644
index 000000000..28dbddf78
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_bn_humanities_tasks
+task: global_mmlu_full_bn_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_world_history.yaml
new file mode 100644
index 000000000..fc8aec7da
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_bn_humanities_tasks
+task: global_mmlu_full_bn_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_human_aging.yaml
new file mode 100644
index 000000000..16a3c204a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_bn_other_tasks
+task: global_mmlu_full_bn_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_human_sexuality.yaml
new file mode 100644
index 000000000..a25244a55
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_bn_social_sciences_tasks
+task: global_mmlu_full_bn_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_international_law.yaml
new file mode 100644
index 000000000..b47f516de
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_bn_humanities_tasks
+task: global_mmlu_full_bn_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_jurisprudence.yaml
new file mode 100644
index 000000000..08ab51e2f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_bn_humanities_tasks
+task: global_mmlu_full_bn_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_logical_fallacies.yaml
new file mode 100644
index 000000000..0885a1a99
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_bn_humanities_tasks
+task: global_mmlu_full_bn_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_machine_learning.yaml
new file mode 100644
index 000000000..f0eb0997d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_bn_stem_tasks
+task: global_mmlu_full_bn_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_management.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_management.yaml
new file mode 100644
index 000000000..d006b4116
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_bn_other_tasks
+task: global_mmlu_full_bn_management
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_marketing.yaml
new file mode 100644
index 000000000..520f94699
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_bn_other_tasks
+task: global_mmlu_full_bn_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_medical_genetics.yaml
new file mode 100644
index 000000000..88caa977d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_bn_other_tasks
+task: global_mmlu_full_bn_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_miscellaneous.yaml
new file mode 100644
index 000000000..9ce31f7b1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_bn_other_tasks
+task: global_mmlu_full_bn_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_moral_disputes.yaml
new file mode 100644
index 000000000..444032164
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_bn_humanities_tasks
+task: global_mmlu_full_bn_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_moral_scenarios.yaml
new file mode 100644
index 000000000..e5f59e15c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_bn_humanities_tasks
+task: global_mmlu_full_bn_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_nutrition.yaml
new file mode 100644
index 000000000..422bba550
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_bn_other_tasks
+task: global_mmlu_full_bn_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_philosophy.yaml
new file mode 100644
index 000000000..62af532bf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_bn_humanities_tasks
+task: global_mmlu_full_bn_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_prehistory.yaml
new file mode 100644
index 000000000..dc49d36c3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_bn_humanities_tasks
+task: global_mmlu_full_bn_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_accounting.yaml
new file mode 100644
index 000000000..bf72a6a42
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_bn_other_tasks
+task: global_mmlu_full_bn_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_law.yaml
new file mode 100644
index 000000000..f49fb1426
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_bn_humanities_tasks
+task: global_mmlu_full_bn_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_medicine.yaml
new file mode 100644
index 000000000..3c53d77a7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_bn_other_tasks
+task: global_mmlu_full_bn_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_psychology.yaml
new file mode 100644
index 000000000..a50c5cbf1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_bn_social_sciences_tasks
+task: global_mmlu_full_bn_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_public_relations.yaml
new file mode 100644
index 000000000..00e2742a4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_bn_social_sciences_tasks
+task: global_mmlu_full_bn_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_security_studies.yaml
new file mode 100644
index 000000000..5a0e7612c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_bn_social_sciences_tasks
+task: global_mmlu_full_bn_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_sociology.yaml
new file mode 100644
index 000000000..e88203197
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_bn_social_sciences_tasks
+task: global_mmlu_full_bn_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_us_foreign_policy.yaml
new file mode 100644
index 000000000..42be796a3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_bn_social_sciences_tasks
+task: global_mmlu_full_bn_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_virology.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_virology.yaml
new file mode 100644
index 000000000..3959f0062
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_bn_other_tasks
+task: global_mmlu_full_bn_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_world_religions.yaml
new file mode 100644
index 000000000..15ee9efc2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _bn_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_bn_humanities_tasks
+task: global_mmlu_full_bn_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/bn/utils.py b/lmms_eval/tasks/global_mmlu/full/bn/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/bn/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/_cs_template_yaml b/lmms_eval/tasks/global_mmlu/full/cs/_cs_template_yaml
new file mode 100644
index 000000000..ce2189a0c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/_cs_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: cs
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs.yaml b/lmms_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs.yaml
new file mode 100644
index 000000000..419d5de4f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_cs
+task:
+  - global_mmlu_full_cs_stem
+  - global_mmlu_full_cs_other
+  - global_mmlu_full_cs_social_sciences
+  - global_mmlu_full_cs_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_humanities.yaml
new file mode 100644
index 000000000..b4b4aff39
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_cs_humanities
+task:
+  - global_mmlu_full_cs_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_other.yaml b/lmms_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_other.yaml
new file mode 100644
index 000000000..302912e45
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_cs_other
+task:
+  - global_mmlu_full_cs_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_social_sciences.yaml
new file mode 100644
index 000000000..d3fed76e4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_cs_social_sciences
+task:
+  - global_mmlu_full_cs_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_stem.yaml b/lmms_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_stem.yaml
new file mode 100644
index 000000000..898bb0927
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_cs_stem
+task:
+  - global_mmlu_full_cs_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_abstract_algebra.yaml
new file mode 100644
index 000000000..40431ec99
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_cs_stem_tasks
+task: global_mmlu_full_cs_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_anatomy.yaml
new file mode 100644
index 000000000..97d7354d4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_cs_stem_tasks
+task: global_mmlu_full_cs_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_astronomy.yaml
new file mode 100644
index 000000000..5b5a5f993
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_cs_stem_tasks
+task: global_mmlu_full_cs_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_business_ethics.yaml
new file mode 100644
index 000000000..6db79c522
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_cs_other_tasks
+task: global_mmlu_full_cs_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_clinical_knowledge.yaml
new file mode 100644
index 000000000..3a17c6051
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_cs_other_tasks
+task: global_mmlu_full_cs_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_biology.yaml
new file mode 100644
index 000000000..9c6597b4c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_cs_stem_tasks
+task: global_mmlu_full_cs_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_chemistry.yaml
new file mode 100644
index 000000000..713af5c32
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_cs_stem_tasks
+task: global_mmlu_full_cs_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_computer_science.yaml
new file mode 100644
index 000000000..fd619d13f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_cs_stem_tasks
+task: global_mmlu_full_cs_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_mathematics.yaml
new file mode 100644
index 000000000..e09563f92
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_cs_stem_tasks
+task: global_mmlu_full_cs_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_medicine.yaml
new file mode 100644
index 000000000..f7b868c92
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_cs_other_tasks
+task: global_mmlu_full_cs_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_physics.yaml
new file mode 100644
index 000000000..e98df3391
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_cs_stem_tasks
+task: global_mmlu_full_cs_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_computer_security.yaml
new file mode 100644
index 000000000..7256ad679
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_cs_stem_tasks
+task: global_mmlu_full_cs_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_conceptual_physics.yaml
new file mode 100644
index 000000000..9bd644989
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_cs_stem_tasks
+task: global_mmlu_full_cs_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_econometrics.yaml
new file mode 100644
index 000000000..c954d3202
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_cs_social_sciences_tasks
+task: global_mmlu_full_cs_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_electrical_engineering.yaml
new file mode 100644
index 000000000..2f80e8ac0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_cs_stem_tasks
+task: global_mmlu_full_cs_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_elementary_mathematics.yaml
new file mode 100644
index 000000000..bfbc2c9d6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_cs_stem_tasks
+task: global_mmlu_full_cs_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_formal_logic.yaml
new file mode 100644
index 000000000..0c2ec8bdc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_cs_humanities_tasks
+task: global_mmlu_full_cs_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_global_facts.yaml
new file mode 100644
index 000000000..6302b417e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_cs_other_tasks
+task: global_mmlu_full_cs_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_biology.yaml
new file mode 100644
index 000000000..b69e9ac39
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_cs_stem_tasks
+task: global_mmlu_full_cs_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_chemistry.yaml
new file mode 100644
index 000000000..67f53cf54
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_cs_stem_tasks
+task: global_mmlu_full_cs_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_computer_science.yaml
new file mode 100644
index 000000000..0be19221b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_cs_stem_tasks
+task: global_mmlu_full_cs_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_european_history.yaml
new file mode 100644
index 000000000..7fa264c36
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_cs_humanities_tasks
+task: global_mmlu_full_cs_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_geography.yaml
new file mode 100644
index 000000000..b9f903c2f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_cs_social_sciences_tasks
+task: global_mmlu_full_cs_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..5bde4d695
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_cs_social_sciences_tasks
+task: global_mmlu_full_cs_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..bb5068edb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_cs_social_sciences_tasks
+task: global_mmlu_full_cs_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_mathematics.yaml
new file mode 100644
index 000000000..87cb3e577
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_cs_stem_tasks
+task: global_mmlu_full_cs_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_microeconomics.yaml
new file mode 100644
index 000000000..33c2e18c8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_cs_social_sciences_tasks
+task: global_mmlu_full_cs_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_physics.yaml
new file mode 100644
index 000000000..1ed095bf4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_cs_stem_tasks
+task: global_mmlu_full_cs_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_psychology.yaml
new file mode 100644
index 000000000..59b623053
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_cs_social_sciences_tasks
+task: global_mmlu_full_cs_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_statistics.yaml
new file mode 100644
index 000000000..1a18ee25b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_cs_stem_tasks
+task: global_mmlu_full_cs_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_us_history.yaml
new file mode 100644
index 000000000..d8d0a2716
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_cs_humanities_tasks
+task: global_mmlu_full_cs_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_world_history.yaml
new file mode 100644
index 000000000..070123060
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_cs_humanities_tasks
+task: global_mmlu_full_cs_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_human_aging.yaml
new file mode 100644
index 000000000..e3f5c7c39
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_cs_other_tasks
+task: global_mmlu_full_cs_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_human_sexuality.yaml
new file mode 100644
index 000000000..61d405c73
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_cs_social_sciences_tasks
+task: global_mmlu_full_cs_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_international_law.yaml
new file mode 100644
index 000000000..509ebee46
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_cs_humanities_tasks
+task: global_mmlu_full_cs_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_jurisprudence.yaml
new file mode 100644
index 000000000..c0e27957b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_cs_humanities_tasks
+task: global_mmlu_full_cs_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_logical_fallacies.yaml
new file mode 100644
index 000000000..85010f3c6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_cs_humanities_tasks
+task: global_mmlu_full_cs_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_machine_learning.yaml
new file mode 100644
index 000000000..32aaa1a65
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_cs_stem_tasks
+task: global_mmlu_full_cs_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_management.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_management.yaml
new file mode 100644
index 000000000..4e1a3a7cb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_cs_other_tasks
+task: global_mmlu_full_cs_management
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_marketing.yaml
new file mode 100644
index 000000000..239e3c0c4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_cs_other_tasks
+task: global_mmlu_full_cs_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_medical_genetics.yaml
new file mode 100644
index 000000000..1c76fee78
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_cs_other_tasks
+task: global_mmlu_full_cs_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_miscellaneous.yaml
new file mode 100644
index 000000000..4be6207a2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_cs_other_tasks
+task: global_mmlu_full_cs_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_moral_disputes.yaml
new file mode 100644
index 000000000..b263f67ec
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_cs_humanities_tasks
+task: global_mmlu_full_cs_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_moral_scenarios.yaml
new file mode 100644
index 000000000..6532a43ed
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_cs_humanities_tasks
+task: global_mmlu_full_cs_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_nutrition.yaml
new file mode 100644
index 000000000..3f04fbcd1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_cs_other_tasks
+task: global_mmlu_full_cs_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_philosophy.yaml
new file mode 100644
index 000000000..2f5093f92
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_cs_humanities_tasks
+task: global_mmlu_full_cs_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_prehistory.yaml
new file mode 100644
index 000000000..a8f5f5a5d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_cs_humanities_tasks
+task: global_mmlu_full_cs_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_accounting.yaml
new file mode 100644
index 000000000..bccb71b23
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_cs_other_tasks
+task: global_mmlu_full_cs_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_law.yaml
new file mode 100644
index 000000000..ff50f50c5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_cs_humanities_tasks
+task: global_mmlu_full_cs_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_medicine.yaml
new file mode 100644
index 000000000..9b8293790
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_cs_other_tasks
+task: global_mmlu_full_cs_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_psychology.yaml
new file mode 100644
index 000000000..e41edb29e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_cs_social_sciences_tasks
+task: global_mmlu_full_cs_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_public_relations.yaml
new file mode 100644
index 000000000..e8fb512db
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_cs_social_sciences_tasks
+task: global_mmlu_full_cs_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_security_studies.yaml
new file mode 100644
index 000000000..64ec0b3f0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_cs_social_sciences_tasks
+task: global_mmlu_full_cs_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_sociology.yaml
new file mode 100644
index 000000000..18214f7c2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_cs_social_sciences_tasks
+task: global_mmlu_full_cs_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_us_foreign_policy.yaml
new file mode 100644
index 000000000..ac42b0970
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_cs_social_sciences_tasks
+task: global_mmlu_full_cs_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_virology.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_virology.yaml
new file mode 100644
index 000000000..a51b8aefa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_cs_other_tasks
+task: global_mmlu_full_cs_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_world_religions.yaml
new file mode 100644
index 000000000..cf9af3e9c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _cs_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_cs_humanities_tasks
+task: global_mmlu_full_cs_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/cs/utils.py b/lmms_eval/tasks/global_mmlu/full/cs/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/cs/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/de/_de_template_yaml b/lmms_eval/tasks/global_mmlu/full/de/_de_template_yaml
new file mode 100644
index 000000000..036b86192
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/_de_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: de
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/de/_global_mmlu_full_de.yaml b/lmms_eval/tasks/global_mmlu/full/de/_global_mmlu_full_de.yaml
new file mode 100644
index 000000000..5217599a6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/_global_mmlu_full_de.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_de
+task:
+  - global_mmlu_full_de_stem
+  - global_mmlu_full_de_other
+  - global_mmlu_full_de_social_sciences
+  - global_mmlu_full_de_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/de/_global_mmlu_full_de_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/de/_global_mmlu_full_de_humanities.yaml
new file mode 100644
index 000000000..df571c67e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/_global_mmlu_full_de_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_de_humanities
+task:
+  - global_mmlu_full_de_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/de/_global_mmlu_full_de_other.yaml b/lmms_eval/tasks/global_mmlu/full/de/_global_mmlu_full_de_other.yaml
new file mode 100644
index 000000000..bfff864e4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/_global_mmlu_full_de_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_de_other
+task:
+  - global_mmlu_full_de_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/de/_global_mmlu_full_de_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/de/_global_mmlu_full_de_social_sciences.yaml
new file mode 100644
index 000000000..8cf304a2c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/_global_mmlu_full_de_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_de_social_sciences
+task:
+  - global_mmlu_full_de_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/de/_global_mmlu_full_de_stem.yaml b/lmms_eval/tasks/global_mmlu/full/de/_global_mmlu_full_de_stem.yaml
new file mode 100644
index 000000000..75d1aa5a1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/_global_mmlu_full_de_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_de_stem
+task:
+  - global_mmlu_full_de_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_abstract_algebra.yaml
new file mode 100644
index 000000000..07cd23566
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_de_stem_tasks
+task: global_mmlu_full_de_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_anatomy.yaml
new file mode 100644
index 000000000..9deb16a6e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_de_stem_tasks
+task: global_mmlu_full_de_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_astronomy.yaml
new file mode 100644
index 000000000..6a743d455
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_de_stem_tasks
+task: global_mmlu_full_de_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_business_ethics.yaml
new file mode 100644
index 000000000..37bf9d454
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_de_other_tasks
+task: global_mmlu_full_de_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_clinical_knowledge.yaml
new file mode 100644
index 000000000..c5ad878a8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_de_other_tasks
+task: global_mmlu_full_de_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_biology.yaml
new file mode 100644
index 000000000..200f9239f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_de_stem_tasks
+task: global_mmlu_full_de_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_chemistry.yaml
new file mode 100644
index 000000000..2bbc4d463
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_de_stem_tasks
+task: global_mmlu_full_de_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_computer_science.yaml
new file mode 100644
index 000000000..ac903e3a5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_de_stem_tasks
+task: global_mmlu_full_de_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_mathematics.yaml
new file mode 100644
index 000000000..616010cad
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_de_stem_tasks
+task: global_mmlu_full_de_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_medicine.yaml
new file mode 100644
index 000000000..b9648ce88
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_de_other_tasks
+task: global_mmlu_full_de_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_physics.yaml
new file mode 100644
index 000000000..d3bc68925
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_de_stem_tasks
+task: global_mmlu_full_de_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_computer_security.yaml
new file mode 100644
index 000000000..fee01f9e1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_de_stem_tasks
+task: global_mmlu_full_de_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_conceptual_physics.yaml
new file mode 100644
index 000000000..201c17d7c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_de_stem_tasks
+task: global_mmlu_full_de_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_econometrics.yaml
new file mode 100644
index 000000000..1d902c3c2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_de_social_sciences_tasks
+task: global_mmlu_full_de_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_electrical_engineering.yaml
new file mode 100644
index 000000000..8dcb6c48b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_de_stem_tasks
+task: global_mmlu_full_de_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_elementary_mathematics.yaml
new file mode 100644
index 000000000..a1ca41ce6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_de_stem_tasks
+task: global_mmlu_full_de_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_formal_logic.yaml
new file mode 100644
index 000000000..6e16729e7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_de_humanities_tasks
+task: global_mmlu_full_de_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_global_facts.yaml
new file mode 100644
index 000000000..a7b092892
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_de_other_tasks
+task: global_mmlu_full_de_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_biology.yaml
new file mode 100644
index 000000000..0ad59551a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_de_stem_tasks
+task: global_mmlu_full_de_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_chemistry.yaml
new file mode 100644
index 000000000..6c0fbd556
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_de_stem_tasks
+task: global_mmlu_full_de_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_computer_science.yaml
new file mode 100644
index 000000000..0aea5ada3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_de_stem_tasks
+task: global_mmlu_full_de_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_european_history.yaml
new file mode 100644
index 000000000..97293b498
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_de_humanities_tasks
+task: global_mmlu_full_de_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_geography.yaml
new file mode 100644
index 000000000..d26a65d97
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_de_social_sciences_tasks
+task: global_mmlu_full_de_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..b6ec78e69
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_de_social_sciences_tasks
+task: global_mmlu_full_de_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..53489d855
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_de_social_sciences_tasks
+task: global_mmlu_full_de_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_mathematics.yaml
new file mode 100644
index 000000000..44a5666f7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_de_stem_tasks
+task: global_mmlu_full_de_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_microeconomics.yaml
new file mode 100644
index 000000000..3b911297c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_de_social_sciences_tasks
+task: global_mmlu_full_de_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_physics.yaml
new file mode 100644
index 000000000..8d17d047e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_de_stem_tasks
+task: global_mmlu_full_de_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_psychology.yaml
new file mode 100644
index 000000000..ae7680027
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_de_social_sciences_tasks
+task: global_mmlu_full_de_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_statistics.yaml
new file mode 100644
index 000000000..4c272287e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_de_stem_tasks
+task: global_mmlu_full_de_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_us_history.yaml
new file mode 100644
index 000000000..9c1eff811
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_de_humanities_tasks
+task: global_mmlu_full_de_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_world_history.yaml
new file mode 100644
index 000000000..11f804a62
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_de_humanities_tasks
+task: global_mmlu_full_de_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_human_aging.yaml
new file mode 100644
index 000000000..7d5b4d775
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_de_other_tasks
+task: global_mmlu_full_de_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_human_sexuality.yaml
new file mode 100644
index 000000000..b3f09c7fb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_de_social_sciences_tasks
+task: global_mmlu_full_de_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_international_law.yaml
new file mode 100644
index 000000000..34bb59185
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_de_humanities_tasks
+task: global_mmlu_full_de_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_jurisprudence.yaml
new file mode 100644
index 000000000..585e99b3c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_de_humanities_tasks
+task: global_mmlu_full_de_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_logical_fallacies.yaml
new file mode 100644
index 000000000..dd09d6ad1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_de_humanities_tasks
+task: global_mmlu_full_de_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_machine_learning.yaml
new file mode 100644
index 000000000..dfe82a9b6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_de_stem_tasks
+task: global_mmlu_full_de_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_management.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_management.yaml
new file mode 100644
index 000000000..7304da38e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_de_other_tasks
+task: global_mmlu_full_de_management
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_marketing.yaml
new file mode 100644
index 000000000..2143e4f17
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_de_other_tasks
+task: global_mmlu_full_de_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_medical_genetics.yaml
new file mode 100644
index 000000000..015498683
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_de_other_tasks
+task: global_mmlu_full_de_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_miscellaneous.yaml
new file mode 100644
index 000000000..0c8bd533b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_de_other_tasks
+task: global_mmlu_full_de_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_moral_disputes.yaml
new file mode 100644
index 000000000..f03361ae6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_de_humanities_tasks
+task: global_mmlu_full_de_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_moral_scenarios.yaml
new file mode 100644
index 000000000..a36519a73
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_de_humanities_tasks
+task: global_mmlu_full_de_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_nutrition.yaml
new file mode 100644
index 000000000..799065cb7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_de_other_tasks
+task: global_mmlu_full_de_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_philosophy.yaml
new file mode 100644
index 000000000..a5f0372b5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_de_humanities_tasks
+task: global_mmlu_full_de_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_prehistory.yaml
new file mode 100644
index 000000000..2145e87d4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_de_humanities_tasks
+task: global_mmlu_full_de_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_accounting.yaml
new file mode 100644
index 000000000..7ad55e977
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_de_other_tasks
+task: global_mmlu_full_de_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_law.yaml
new file mode 100644
index 000000000..6f4e338fa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_de_humanities_tasks
+task: global_mmlu_full_de_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_medicine.yaml
new file mode 100644
index 000000000..7a1214a6c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_de_other_tasks
+task: global_mmlu_full_de_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_psychology.yaml
new file mode 100644
index 000000000..a2d49ec8c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_de_social_sciences_tasks
+task: global_mmlu_full_de_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_public_relations.yaml
new file mode 100644
index 000000000..4b7d23a83
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_de_social_sciences_tasks
+task: global_mmlu_full_de_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_security_studies.yaml
new file mode 100644
index 000000000..a1a3b22ee
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_de_social_sciences_tasks
+task: global_mmlu_full_de_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_sociology.yaml
new file mode 100644
index 000000000..fefef9d01
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_de_social_sciences_tasks
+task: global_mmlu_full_de_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_us_foreign_policy.yaml
new file mode 100644
index 000000000..35394ab5b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_de_social_sciences_tasks
+task: global_mmlu_full_de_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_virology.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_virology.yaml
new file mode 100644
index 000000000..f0f2f5957
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_de_other_tasks
+task: global_mmlu_full_de_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_world_religions.yaml
new file mode 100644
index 000000000..1a43e6fe0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _de_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_de_humanities_tasks
+task: global_mmlu_full_de_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/de/utils.py b/lmms_eval/tasks/global_mmlu/full/de/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/de/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/el/_el_template_yaml b/lmms_eval/tasks/global_mmlu/full/el/_el_template_yaml
new file mode 100644
index 000000000..5fccad5ec
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/_el_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: el
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el.yaml b/lmms_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el.yaml
new file mode 100644
index 000000000..9d44d8c8b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_el
+task:
+  - global_mmlu_full_el_stem
+  - global_mmlu_full_el_other
+  - global_mmlu_full_el_social_sciences
+  - global_mmlu_full_el_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_humanities.yaml
new file mode 100644
index 000000000..f07f2b524
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_el_humanities
+task:
+  - global_mmlu_full_el_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_other.yaml b/lmms_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_other.yaml
new file mode 100644
index 000000000..938292f09
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_el_other
+task:
+  - global_mmlu_full_el_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_social_sciences.yaml
new file mode 100644
index 000000000..e72e1e9c8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_el_social_sciences
+task:
+  - global_mmlu_full_el_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_stem.yaml b/lmms_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_stem.yaml
new file mode 100644
index 000000000..2123be088
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_el_stem
+task:
+  - global_mmlu_full_el_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_abstract_algebra.yaml
new file mode 100644
index 000000000..bc56c0696
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_el_stem_tasks
+task: global_mmlu_full_el_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_anatomy.yaml
new file mode 100644
index 000000000..0b2e0e7ff
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_el_stem_tasks
+task: global_mmlu_full_el_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_astronomy.yaml
new file mode 100644
index 000000000..7faf7389a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_el_stem_tasks
+task: global_mmlu_full_el_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_business_ethics.yaml
new file mode 100644
index 000000000..0e8b5bb43
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_el_other_tasks
+task: global_mmlu_full_el_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_clinical_knowledge.yaml
new file mode 100644
index 000000000..51ade4218
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_el_other_tasks
+task: global_mmlu_full_el_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_biology.yaml
new file mode 100644
index 000000000..cf3aa362c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_el_stem_tasks
+task: global_mmlu_full_el_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_chemistry.yaml
new file mode 100644
index 000000000..cd8e1dacb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_el_stem_tasks
+task: global_mmlu_full_el_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_computer_science.yaml
new file mode 100644
index 000000000..f1ea08597
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_el_stem_tasks
+task: global_mmlu_full_el_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_mathematics.yaml
new file mode 100644
index 000000000..0ec055b7f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_el_stem_tasks
+task: global_mmlu_full_el_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_medicine.yaml
new file mode 100644
index 000000000..b16b545b0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_el_other_tasks
+task: global_mmlu_full_el_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_physics.yaml
new file mode 100644
index 000000000..a4630f7f8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_el_stem_tasks
+task: global_mmlu_full_el_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_computer_security.yaml
new file mode 100644
index 000000000..a40228ea3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_el_stem_tasks
+task: global_mmlu_full_el_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_conceptual_physics.yaml
new file mode 100644
index 000000000..e7baf6e60
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_el_stem_tasks
+task: global_mmlu_full_el_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_econometrics.yaml
new file mode 100644
index 000000000..48e590213
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_el_social_sciences_tasks
+task: global_mmlu_full_el_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_electrical_engineering.yaml
new file mode 100644
index 000000000..294c3c5db
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_el_stem_tasks
+task: global_mmlu_full_el_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_elementary_mathematics.yaml
new file mode 100644
index 000000000..4373d82e3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_el_stem_tasks
+task: global_mmlu_full_el_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_formal_logic.yaml
new file mode 100644
index 000000000..81799a17c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_el_humanities_tasks
+task: global_mmlu_full_el_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_global_facts.yaml
new file mode 100644
index 000000000..6317eeec7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_el_other_tasks
+task: global_mmlu_full_el_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_biology.yaml
new file mode 100644
index 000000000..fa5958aa1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_el_stem_tasks
+task: global_mmlu_full_el_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_chemistry.yaml
new file mode 100644
index 000000000..38053add3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_el_stem_tasks
+task: global_mmlu_full_el_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_computer_science.yaml
new file mode 100644
index 000000000..4fe73214d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_el_stem_tasks
+task: global_mmlu_full_el_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_european_history.yaml
new file mode 100644
index 000000000..8beb1e3f5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_el_humanities_tasks
+task: global_mmlu_full_el_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_geography.yaml
new file mode 100644
index 000000000..22c083214
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_el_social_sciences_tasks
+task: global_mmlu_full_el_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..e3b330411
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_el_social_sciences_tasks
+task: global_mmlu_full_el_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..63ba6a054
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_el_social_sciences_tasks
+task: global_mmlu_full_el_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_mathematics.yaml
new file mode 100644
index 000000000..f6ff6e2c0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_el_stem_tasks
+task: global_mmlu_full_el_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_microeconomics.yaml
new file mode 100644
index 000000000..5e4deeebc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_el_social_sciences_tasks
+task: global_mmlu_full_el_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_physics.yaml
new file mode 100644
index 000000000..cb8757033
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_el_stem_tasks
+task: global_mmlu_full_el_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_psychology.yaml
new file mode 100644
index 000000000..e82d1b530
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_el_social_sciences_tasks
+task: global_mmlu_full_el_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_statistics.yaml
new file mode 100644
index 000000000..0003184cc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_el_stem_tasks
+task: global_mmlu_full_el_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_us_history.yaml
new file mode 100644
index 000000000..f5e0a3675
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_el_humanities_tasks
+task: global_mmlu_full_el_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_world_history.yaml
new file mode 100644
index 000000000..ac460ea8b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_el_humanities_tasks
+task: global_mmlu_full_el_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_human_aging.yaml
new file mode 100644
index 000000000..8a40e04f2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_el_other_tasks
+task: global_mmlu_full_el_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_human_sexuality.yaml
new file mode 100644
index 000000000..de5075bcf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_el_social_sciences_tasks
+task: global_mmlu_full_el_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_international_law.yaml
new file mode 100644
index 000000000..2fb93f2c8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_el_humanities_tasks
+task: global_mmlu_full_el_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_jurisprudence.yaml
new file mode 100644
index 000000000..624e040ad
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_el_humanities_tasks
+task: global_mmlu_full_el_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_logical_fallacies.yaml
new file mode 100644
index 000000000..7e5bdb4f3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_el_humanities_tasks
+task: global_mmlu_full_el_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_machine_learning.yaml
new file mode 100644
index 000000000..180f3b25e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_el_stem_tasks
+task: global_mmlu_full_el_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_management.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_management.yaml
new file mode 100644
index 000000000..40487fb17
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_el_other_tasks
+task: global_mmlu_full_el_management
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_marketing.yaml
new file mode 100644
index 000000000..781d41708
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_el_other_tasks
+task: global_mmlu_full_el_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_medical_genetics.yaml
new file mode 100644
index 000000000..2ca011464
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_el_other_tasks
+task: global_mmlu_full_el_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_miscellaneous.yaml
new file mode 100644
index 000000000..66114367b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_el_other_tasks
+task: global_mmlu_full_el_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_moral_disputes.yaml
new file mode 100644
index 000000000..c553ab7be
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_el_humanities_tasks
+task: global_mmlu_full_el_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_moral_scenarios.yaml
new file mode 100644
index 000000000..14a79a4a3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_el_humanities_tasks
+task: global_mmlu_full_el_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_nutrition.yaml
new file mode 100644
index 000000000..595daa39f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_el_other_tasks
+task: global_mmlu_full_el_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_philosophy.yaml
new file mode 100644
index 000000000..25b121b69
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_el_humanities_tasks
+task: global_mmlu_full_el_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_prehistory.yaml
new file mode 100644
index 000000000..5938a174e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_el_humanities_tasks
+task: global_mmlu_full_el_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_accounting.yaml
new file mode 100644
index 000000000..002b02aa9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_el_other_tasks
+task: global_mmlu_full_el_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_law.yaml
new file mode 100644
index 000000000..7b4570380
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_el_humanities_tasks
+task: global_mmlu_full_el_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_medicine.yaml
new file mode 100644
index 000000000..a31d4e3bb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_el_other_tasks
+task: global_mmlu_full_el_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_psychology.yaml
new file mode 100644
index 000000000..6e0480799
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_el_social_sciences_tasks
+task: global_mmlu_full_el_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_public_relations.yaml
new file mode 100644
index 000000000..264799d61
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_el_social_sciences_tasks
+task: global_mmlu_full_el_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_security_studies.yaml
new file mode 100644
index 000000000..19ffae47e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_el_social_sciences_tasks
+task: global_mmlu_full_el_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_sociology.yaml
new file mode 100644
index 000000000..f57d3e0aa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_el_social_sciences_tasks
+task: global_mmlu_full_el_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_us_foreign_policy.yaml
new file mode 100644
index 000000000..14c764407
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_el_social_sciences_tasks
+task: global_mmlu_full_el_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_virology.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_virology.yaml
new file mode 100644
index 000000000..0e4443589
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_el_other_tasks
+task: global_mmlu_full_el_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_world_religions.yaml
new file mode 100644
index 000000000..60f8e52e3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _el_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_el_humanities_tasks
+task: global_mmlu_full_el_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/el/utils.py b/lmms_eval/tasks/global_mmlu/full/el/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/el/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/en/_en_template_yaml b/lmms_eval/tasks/global_mmlu/full/en/_en_template_yaml
new file mode 100644
index 000000000..3690afa28
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/_en_template_yaml
@@ -0,0 +1,25 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: en
+test_split: test
+# fewshot_split: dev
+training_split: dev
+validation_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    # higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    # higher_is_better: true
+metadata:
+  version: 0.0
+
+# TO CHECK
+# should_decontaminate: true
+# doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
\ No newline at end of file
diff --git a/lmms_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en.yaml b/lmms_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en.yaml
new file mode 100644
index 000000000..57748eef5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en.yaml
@@ -0,0 +1,12 @@
+group: global_mmlu_full_en
+task:
+  - global_mmlu_full_en_stem
+  - global_mmlu_full_en_other
+  - global_mmlu_full_en_social_sciences
+  - global_mmlu_full_en_humanities
+aggregate_metric_list:
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_humanities.yaml
new file mode 100644
index 000000000..4455fbcfe
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_en_humanities
+task:
+  - global_mmlu_full_en_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_other.yaml b/lmms_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_other.yaml
new file mode 100644
index 000000000..cca60e529
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_en_other
+task:
+  - global_mmlu_full_en_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_social_sciences.yaml
new file mode 100644
index 000000000..becac7a59
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_en_social_sciences
+task:
+  - global_mmlu_full_en_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_stem.yaml b/lmms_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_stem.yaml
new file mode 100644
index 000000000..71aac0612
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_en_stem
+task:
+  - global_mmlu_full_en_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_abstract_algebra.yaml
new file mode 100644
index 000000000..3d7a5ed80
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_en_stem_tasks
+task: global_mmlu_full_en_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_anatomy.yaml
new file mode 100644
index 000000000..f2267ad88
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_en_stem_tasks
+task: global_mmlu_full_en_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_astronomy.yaml
new file mode 100644
index 000000000..6999c30f6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_en_stem_tasks
+task: global_mmlu_full_en_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_business_ethics.yaml
new file mode 100644
index 000000000..56a6e490e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_en_other_tasks
+task: global_mmlu_full_en_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_clinical_knowledge.yaml
new file mode 100644
index 000000000..60425fadc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_en_other_tasks
+task: global_mmlu_full_en_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_biology.yaml
new file mode 100644
index 000000000..9b5f2f8c5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_en_stem_tasks
+task: global_mmlu_full_en_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_chemistry.yaml
new file mode 100644
index 000000000..8e2ab91f9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_en_stem_tasks
+task: global_mmlu_full_en_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_computer_science.yaml
new file mode 100644
index 000000000..9abf38dbd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_en_stem_tasks
+task: global_mmlu_full_en_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_mathematics.yaml
new file mode 100644
index 000000000..5da6199fb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_en_stem_tasks
+task: global_mmlu_full_en_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_medicine.yaml
new file mode 100644
index 000000000..c568f36b7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_en_other_tasks
+task: global_mmlu_full_en_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_physics.yaml
new file mode 100644
index 000000000..ac0440196
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_en_stem_tasks
+task: global_mmlu_full_en_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_computer_security.yaml
new file mode 100644
index 000000000..be47dbde0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_en_stem_tasks
+task: global_mmlu_full_en_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_conceptual_physics.yaml
new file mode 100644
index 000000000..86180924f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_en_stem_tasks
+task: global_mmlu_full_en_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_econometrics.yaml
new file mode 100644
index 000000000..a75d329f4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_en_social_sciences_tasks
+task: global_mmlu_full_en_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_electrical_engineering.yaml
new file mode 100644
index 000000000..2568993f2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_en_stem_tasks
+task: global_mmlu_full_en_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_elementary_mathematics.yaml
new file mode 100644
index 000000000..622a99f83
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_en_stem_tasks
+task: global_mmlu_full_en_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_formal_logic.yaml
new file mode 100644
index 000000000..109ca44ae
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_en_humanities_tasks
+task: global_mmlu_full_en_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_global_facts.yaml
new file mode 100644
index 000000000..39daa506b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_en_other_tasks
+task: global_mmlu_full_en_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_biology.yaml
new file mode 100644
index 000000000..063392eb2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_en_stem_tasks
+task: global_mmlu_full_en_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_chemistry.yaml
new file mode 100644
index 000000000..452e9445d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_en_stem_tasks
+task: global_mmlu_full_en_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_computer_science.yaml
new file mode 100644
index 000000000..baf431362
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_en_stem_tasks
+task: global_mmlu_full_en_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_european_history.yaml
new file mode 100644
index 000000000..fceda5c29
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_en_humanities_tasks
+task: global_mmlu_full_en_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_geography.yaml
new file mode 100644
index 000000000..4fbb9ade5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_en_social_sciences_tasks
+task: global_mmlu_full_en_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..73ca9087a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_en_social_sciences_tasks
+task: global_mmlu_full_en_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..1b9ca7a94
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_en_social_sciences_tasks
+task: global_mmlu_full_en_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_mathematics.yaml
new file mode 100644
index 000000000..9be50ad26
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_en_stem_tasks
+task: global_mmlu_full_en_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_microeconomics.yaml
new file mode 100644
index 000000000..d93285cb3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_en_social_sciences_tasks
+task: global_mmlu_full_en_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_physics.yaml
new file mode 100644
index 000000000..2f74c609f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_en_stem_tasks
+task: global_mmlu_full_en_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_psychology.yaml
new file mode 100644
index 000000000..365762ba0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_en_social_sciences_tasks
+task: global_mmlu_full_en_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_statistics.yaml
new file mode 100644
index 000000000..d6ca42ad2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_en_stem_tasks
+task: global_mmlu_full_en_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_us_history.yaml
new file mode 100644
index 000000000..4f20a4ddc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_en_humanities_tasks
+task: global_mmlu_full_en_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_world_history.yaml
new file mode 100644
index 000000000..d0fce4035
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_en_humanities_tasks
+task: global_mmlu_full_en_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_human_aging.yaml
new file mode 100644
index 000000000..35320a854
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_en_other_tasks
+task: global_mmlu_full_en_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_human_sexuality.yaml
new file mode 100644
index 000000000..86096c5d0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_en_social_sciences_tasks
+task: global_mmlu_full_en_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_international_law.yaml
new file mode 100644
index 000000000..8a41e9fcf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_en_humanities_tasks
+task: global_mmlu_full_en_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_jurisprudence.yaml
new file mode 100644
index 000000000..aa34c4433
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_en_humanities_tasks
+task: global_mmlu_full_en_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_logical_fallacies.yaml
new file mode 100644
index 000000000..50c105b48
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_en_humanities_tasks
+task: global_mmlu_full_en_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_machine_learning.yaml
new file mode 100644
index 000000000..35f496c17
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_en_stem_tasks
+task: global_mmlu_full_en_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_management.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_management.yaml
new file mode 100644
index 000000000..d8499d9fa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_en_other_tasks
+task: global_mmlu_full_en_management
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_marketing.yaml
new file mode 100644
index 000000000..05f8f0ec9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_en_other_tasks
+task: global_mmlu_full_en_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_medical_genetics.yaml
new file mode 100644
index 000000000..8f272510f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_en_other_tasks
+task: global_mmlu_full_en_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_miscellaneous.yaml
new file mode 100644
index 000000000..a72fad223
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_en_other_tasks
+task: global_mmlu_full_en_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_moral_disputes.yaml
new file mode 100644
index 000000000..2504abeb6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_en_humanities_tasks
+task: global_mmlu_full_en_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_moral_scenarios.yaml
new file mode 100644
index 000000000..4ae4c37a5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_en_humanities_tasks
+task: global_mmlu_full_en_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_nutrition.yaml
new file mode 100644
index 000000000..b5364f69d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_en_other_tasks
+task: global_mmlu_full_en_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_philosophy.yaml
new file mode 100644
index 000000000..6e68d7e72
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_en_humanities_tasks
+task: global_mmlu_full_en_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_prehistory.yaml
new file mode 100644
index 000000000..72e93368a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_en_humanities_tasks
+task: global_mmlu_full_en_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_accounting.yaml
new file mode 100644
index 000000000..cdb66ead8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_en_other_tasks
+task: global_mmlu_full_en_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_law.yaml
new file mode 100644
index 000000000..671202780
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_en_humanities_tasks
+task: global_mmlu_full_en_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_medicine.yaml
new file mode 100644
index 000000000..ffbcb29b3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_en_other_tasks
+task: global_mmlu_full_en_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_psychology.yaml
new file mode 100644
index 000000000..1abea59b5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_en_social_sciences_tasks
+task: global_mmlu_full_en_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_public_relations.yaml
new file mode 100644
index 000000000..9df4f4918
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_en_social_sciences_tasks
+task: global_mmlu_full_en_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_security_studies.yaml
new file mode 100644
index 000000000..addb6934d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_en_social_sciences_tasks
+task: global_mmlu_full_en_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_sociology.yaml
new file mode 100644
index 000000000..a198cb84c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_en_social_sciences_tasks
+task: global_mmlu_full_en_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_us_foreign_policy.yaml
new file mode 100644
index 000000000..047b61e00
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_en_social_sciences_tasks
+task: global_mmlu_full_en_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_virology.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_virology.yaml
new file mode 100644
index 000000000..bb74fefdd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_en_other_tasks
+task: global_mmlu_full_en_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_world_religions.yaml
new file mode 100644
index 000000000..2c453bf74
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _en_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_en_humanities_tasks
+task: global_mmlu_full_en_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/en/utils.py b/lmms_eval/tasks/global_mmlu/full/en/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/en/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/es/_es_template_yaml b/lmms_eval/tasks/global_mmlu/full/es/_es_template_yaml
new file mode 100644
index 000000000..443af17cc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/_es_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: es
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/es/_global_mmlu_full_es.yaml b/lmms_eval/tasks/global_mmlu/full/es/_global_mmlu_full_es.yaml
new file mode 100644
index 000000000..13d2eccf3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/_global_mmlu_full_es.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_es
+task:
+  - global_mmlu_full_es_stem
+  - global_mmlu_full_es_other
+  - global_mmlu_full_es_social_sciences
+  - global_mmlu_full_es_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/es/_global_mmlu_full_es_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/es/_global_mmlu_full_es_humanities.yaml
new file mode 100644
index 000000000..bda6944e1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/_global_mmlu_full_es_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_es_humanities
+task:
+  - global_mmlu_full_es_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/es/_global_mmlu_full_es_other.yaml b/lmms_eval/tasks/global_mmlu/full/es/_global_mmlu_full_es_other.yaml
new file mode 100644
index 000000000..610366ef7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/_global_mmlu_full_es_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_es_other
+task:
+  - global_mmlu_full_es_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/es/_global_mmlu_full_es_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/es/_global_mmlu_full_es_social_sciences.yaml
new file mode 100644
index 000000000..009486903
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/_global_mmlu_full_es_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_es_social_sciences
+task:
+  - global_mmlu_full_es_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/es/_global_mmlu_full_es_stem.yaml b/lmms_eval/tasks/global_mmlu/full/es/_global_mmlu_full_es_stem.yaml
new file mode 100644
index 000000000..483a8fd6f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/_global_mmlu_full_es_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_es_stem
+task:
+  - global_mmlu_full_es_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_abstract_algebra.yaml
new file mode 100644
index 000000000..02fb72001
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_es_stem_tasks
+task: global_mmlu_full_es_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_anatomy.yaml
new file mode 100644
index 000000000..40f05e7be
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_es_stem_tasks
+task: global_mmlu_full_es_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_astronomy.yaml
new file mode 100644
index 000000000..fb688c13c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_es_stem_tasks
+task: global_mmlu_full_es_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_business_ethics.yaml
new file mode 100644
index 000000000..aab858f1d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_es_other_tasks
+task: global_mmlu_full_es_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_clinical_knowledge.yaml
new file mode 100644
index 000000000..a3483f8d0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_es_other_tasks
+task: global_mmlu_full_es_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_biology.yaml
new file mode 100644
index 000000000..36658ab6c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_es_stem_tasks
+task: global_mmlu_full_es_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_chemistry.yaml
new file mode 100644
index 000000000..47a474449
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_es_stem_tasks
+task: global_mmlu_full_es_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_computer_science.yaml
new file mode 100644
index 000000000..4154324e4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_es_stem_tasks
+task: global_mmlu_full_es_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_mathematics.yaml
new file mode 100644
index 000000000..85bc62614
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_es_stem_tasks
+task: global_mmlu_full_es_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_medicine.yaml
new file mode 100644
index 000000000..40e8d1291
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_es_other_tasks
+task: global_mmlu_full_es_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_physics.yaml
new file mode 100644
index 000000000..7ebc5e950
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_es_stem_tasks
+task: global_mmlu_full_es_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_computer_security.yaml
new file mode 100644
index 000000000..b586eb2b8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_es_stem_tasks
+task: global_mmlu_full_es_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_conceptual_physics.yaml
new file mode 100644
index 000000000..4186cec69
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_es_stem_tasks
+task: global_mmlu_full_es_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_econometrics.yaml
new file mode 100644
index 000000000..3d61c8f9d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_es_social_sciences_tasks
+task: global_mmlu_full_es_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_electrical_engineering.yaml
new file mode 100644
index 000000000..1a454d796
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_es_stem_tasks
+task: global_mmlu_full_es_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_elementary_mathematics.yaml
new file mode 100644
index 000000000..772436e6b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_es_stem_tasks
+task: global_mmlu_full_es_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_formal_logic.yaml
new file mode 100644
index 000000000..da6223fe3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_es_humanities_tasks
+task: global_mmlu_full_es_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_global_facts.yaml
new file mode 100644
index 000000000..ae3b5912b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_es_other_tasks
+task: global_mmlu_full_es_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_biology.yaml
new file mode 100644
index 000000000..79a72140d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_es_stem_tasks
+task: global_mmlu_full_es_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_chemistry.yaml
new file mode 100644
index 000000000..27ba75708
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_es_stem_tasks
+task: global_mmlu_full_es_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_computer_science.yaml
new file mode 100644
index 000000000..72ad45057
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_es_stem_tasks
+task: global_mmlu_full_es_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_european_history.yaml
new file mode 100644
index 000000000..2cec9d5f6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_es_humanities_tasks
+task: global_mmlu_full_es_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_geography.yaml
new file mode 100644
index 000000000..5ee91f710
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_es_social_sciences_tasks
+task: global_mmlu_full_es_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..b3f103199
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_es_social_sciences_tasks
+task: global_mmlu_full_es_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..d555129a4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_es_social_sciences_tasks
+task: global_mmlu_full_es_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_mathematics.yaml
new file mode 100644
index 000000000..a12163361
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_es_stem_tasks
+task: global_mmlu_full_es_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_microeconomics.yaml
new file mode 100644
index 000000000..d4c288443
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_es_social_sciences_tasks
+task: global_mmlu_full_es_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_physics.yaml
new file mode 100644
index 000000000..fb83ad1e4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_es_stem_tasks
+task: global_mmlu_full_es_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_psychology.yaml
new file mode 100644
index 000000000..4bcd53e48
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_es_social_sciences_tasks
+task: global_mmlu_full_es_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_statistics.yaml
new file mode 100644
index 000000000..900936eb1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_es_stem_tasks
+task: global_mmlu_full_es_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_us_history.yaml
new file mode 100644
index 000000000..d54acd65a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_es_humanities_tasks
+task: global_mmlu_full_es_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_world_history.yaml
new file mode 100644
index 000000000..2a654fe86
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_es_humanities_tasks
+task: global_mmlu_full_es_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_human_aging.yaml
new file mode 100644
index 000000000..47bd89009
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_es_other_tasks
+task: global_mmlu_full_es_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_human_sexuality.yaml
new file mode 100644
index 000000000..29925c347
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_es_social_sciences_tasks
+task: global_mmlu_full_es_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_international_law.yaml
new file mode 100644
index 000000000..abe4ef94f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_es_humanities_tasks
+task: global_mmlu_full_es_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_jurisprudence.yaml
new file mode 100644
index 000000000..751878fe1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_es_humanities_tasks
+task: global_mmlu_full_es_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_logical_fallacies.yaml
new file mode 100644
index 000000000..55233f7f2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_es_humanities_tasks
+task: global_mmlu_full_es_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_machine_learning.yaml
new file mode 100644
index 000000000..9a11e310c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_es_stem_tasks
+task: global_mmlu_full_es_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_management.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_management.yaml
new file mode 100644
index 000000000..a31b4c268
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_es_other_tasks
+task: global_mmlu_full_es_management
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_marketing.yaml
new file mode 100644
index 000000000..22136569e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_es_other_tasks
+task: global_mmlu_full_es_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_medical_genetics.yaml
new file mode 100644
index 000000000..18fc7a23e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_es_other_tasks
+task: global_mmlu_full_es_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_miscellaneous.yaml
new file mode 100644
index 000000000..5b3955a93
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_es_other_tasks
+task: global_mmlu_full_es_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_moral_disputes.yaml
new file mode 100644
index 000000000..57095856f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_es_humanities_tasks
+task: global_mmlu_full_es_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_moral_scenarios.yaml
new file mode 100644
index 000000000..ed31f8cc9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_es_humanities_tasks
+task: global_mmlu_full_es_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_nutrition.yaml
new file mode 100644
index 000000000..07746d096
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_es_other_tasks
+task: global_mmlu_full_es_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_philosophy.yaml
new file mode 100644
index 000000000..3853e1622
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_es_humanities_tasks
+task: global_mmlu_full_es_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_prehistory.yaml
new file mode 100644
index 000000000..b75ac9df7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_es_humanities_tasks
+task: global_mmlu_full_es_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_accounting.yaml
new file mode 100644
index 000000000..da8fd46ff
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_es_other_tasks
+task: global_mmlu_full_es_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_law.yaml
new file mode 100644
index 000000000..ddd0ab3d1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_es_humanities_tasks
+task: global_mmlu_full_es_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_medicine.yaml
new file mode 100644
index 000000000..6be1ae817
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_es_other_tasks
+task: global_mmlu_full_es_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_psychology.yaml
new file mode 100644
index 000000000..cadc7f964
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_es_social_sciences_tasks
+task: global_mmlu_full_es_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_public_relations.yaml
new file mode 100644
index 000000000..72609ea90
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_es_social_sciences_tasks
+task: global_mmlu_full_es_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_security_studies.yaml
new file mode 100644
index 000000000..319123c61
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_es_social_sciences_tasks
+task: global_mmlu_full_es_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_sociology.yaml
new file mode 100644
index 000000000..dec44c293
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_es_social_sciences_tasks
+task: global_mmlu_full_es_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_us_foreign_policy.yaml
new file mode 100644
index 000000000..a18a3942d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_es_social_sciences_tasks
+task: global_mmlu_full_es_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_virology.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_virology.yaml
new file mode 100644
index 000000000..b06431e21
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_es_other_tasks
+task: global_mmlu_full_es_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_world_religions.yaml
new file mode 100644
index 000000000..4d9d6b792
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _es_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_es_humanities_tasks
+task: global_mmlu_full_es_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/es/utils.py b/lmms_eval/tasks/global_mmlu/full/es/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/es/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/_fa_template_yaml b/lmms_eval/tasks/global_mmlu/full/fa/_fa_template_yaml
new file mode 100644
index 000000000..952259b2f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/_fa_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: fa
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa.yaml b/lmms_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa.yaml
new file mode 100644
index 000000000..282664e59
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_fa
+task:
+  - global_mmlu_full_fa_stem
+  - global_mmlu_full_fa_other
+  - global_mmlu_full_fa_social_sciences
+  - global_mmlu_full_fa_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_humanities.yaml
new file mode 100644
index 000000000..f36ecea5f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_fa_humanities
+task:
+  - global_mmlu_full_fa_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_other.yaml b/lmms_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_other.yaml
new file mode 100644
index 000000000..dd57bb86b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_fa_other
+task:
+  - global_mmlu_full_fa_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_social_sciences.yaml
new file mode 100644
index 000000000..9e7da860f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_fa_social_sciences
+task:
+  - global_mmlu_full_fa_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_stem.yaml b/lmms_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_stem.yaml
new file mode 100644
index 000000000..5bf2eb013
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_fa_stem
+task:
+  - global_mmlu_full_fa_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_abstract_algebra.yaml
new file mode 100644
index 000000000..1014795f0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_fa_stem_tasks
+task: global_mmlu_full_fa_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_anatomy.yaml
new file mode 100644
index 000000000..317705c92
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_fa_stem_tasks
+task: global_mmlu_full_fa_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_astronomy.yaml
new file mode 100644
index 000000000..454759641
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_fa_stem_tasks
+task: global_mmlu_full_fa_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_business_ethics.yaml
new file mode 100644
index 000000000..3c0dd60b7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_fa_other_tasks
+task: global_mmlu_full_fa_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_clinical_knowledge.yaml
new file mode 100644
index 000000000..a7af0e21f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_fa_other_tasks
+task: global_mmlu_full_fa_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_biology.yaml
new file mode 100644
index 000000000..31ae6d71c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_fa_stem_tasks
+task: global_mmlu_full_fa_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_chemistry.yaml
new file mode 100644
index 000000000..8b099f413
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_fa_stem_tasks
+task: global_mmlu_full_fa_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_computer_science.yaml
new file mode 100644
index 000000000..07491e5bf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_fa_stem_tasks
+task: global_mmlu_full_fa_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_mathematics.yaml
new file mode 100644
index 000000000..774f6b970
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_fa_stem_tasks
+task: global_mmlu_full_fa_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_medicine.yaml
new file mode 100644
index 000000000..13d6f5a2a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_fa_other_tasks
+task: global_mmlu_full_fa_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_physics.yaml
new file mode 100644
index 000000000..1e415b8cf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_fa_stem_tasks
+task: global_mmlu_full_fa_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_computer_security.yaml
new file mode 100644
index 000000000..ae47213b3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_fa_stem_tasks
+task: global_mmlu_full_fa_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_conceptual_physics.yaml
new file mode 100644
index 000000000..c3f2ba4ce
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_fa_stem_tasks
+task: global_mmlu_full_fa_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_econometrics.yaml
new file mode 100644
index 000000000..6cf79a922
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_fa_social_sciences_tasks
+task: global_mmlu_full_fa_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_electrical_engineering.yaml
new file mode 100644
index 000000000..ab7aa8584
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_fa_stem_tasks
+task: global_mmlu_full_fa_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_elementary_mathematics.yaml
new file mode 100644
index 000000000..b83f6ddcf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_fa_stem_tasks
+task: global_mmlu_full_fa_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_formal_logic.yaml
new file mode 100644
index 000000000..cab2effa0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_fa_humanities_tasks
+task: global_mmlu_full_fa_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_global_facts.yaml
new file mode 100644
index 000000000..93d11b75b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_fa_other_tasks
+task: global_mmlu_full_fa_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_biology.yaml
new file mode 100644
index 000000000..59b6869bd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_fa_stem_tasks
+task: global_mmlu_full_fa_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_chemistry.yaml
new file mode 100644
index 000000000..8d15d4b4e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_fa_stem_tasks
+task: global_mmlu_full_fa_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_computer_science.yaml
new file mode 100644
index 000000000..a02df4f13
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_fa_stem_tasks
+task: global_mmlu_full_fa_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_european_history.yaml
new file mode 100644
index 000000000..e18b2c7b6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_fa_humanities_tasks
+task: global_mmlu_full_fa_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_geography.yaml
new file mode 100644
index 000000000..d94c7e89f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_fa_social_sciences_tasks
+task: global_mmlu_full_fa_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..e10078959
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_fa_social_sciences_tasks
+task: global_mmlu_full_fa_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..a9ad06337
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_fa_social_sciences_tasks
+task: global_mmlu_full_fa_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_mathematics.yaml
new file mode 100644
index 000000000..2c733b17d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_fa_stem_tasks
+task: global_mmlu_full_fa_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_microeconomics.yaml
new file mode 100644
index 000000000..4f88f0aa3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_fa_social_sciences_tasks
+task: global_mmlu_full_fa_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_physics.yaml
new file mode 100644
index 000000000..64fdef984
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_fa_stem_tasks
+task: global_mmlu_full_fa_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_psychology.yaml
new file mode 100644
index 000000000..c43a115be
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_fa_social_sciences_tasks
+task: global_mmlu_full_fa_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_statistics.yaml
new file mode 100644
index 000000000..ebb4e82d7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_fa_stem_tasks
+task: global_mmlu_full_fa_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_us_history.yaml
new file mode 100644
index 000000000..a0041e33e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_fa_humanities_tasks
+task: global_mmlu_full_fa_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_world_history.yaml
new file mode 100644
index 000000000..66f38f54b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_fa_humanities_tasks
+task: global_mmlu_full_fa_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_human_aging.yaml
new file mode 100644
index 000000000..95a2addea
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_fa_other_tasks
+task: global_mmlu_full_fa_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_human_sexuality.yaml
new file mode 100644
index 000000000..475a71fdb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_fa_social_sciences_tasks
+task: global_mmlu_full_fa_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_international_law.yaml
new file mode 100644
index 000000000..c0d6aec2b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_fa_humanities_tasks
+task: global_mmlu_full_fa_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_jurisprudence.yaml
new file mode 100644
index 000000000..0d82bd5a6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_fa_humanities_tasks
+task: global_mmlu_full_fa_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_logical_fallacies.yaml
new file mode 100644
index 000000000..a8e89d3c7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_fa_humanities_tasks
+task: global_mmlu_full_fa_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_machine_learning.yaml
new file mode 100644
index 000000000..4e4d1a8a7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_fa_stem_tasks
+task: global_mmlu_full_fa_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_management.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_management.yaml
new file mode 100644
index 000000000..e7e592bae
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_fa_other_tasks
+task: global_mmlu_full_fa_management
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_marketing.yaml
new file mode 100644
index 000000000..c0e7ef1f4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_fa_other_tasks
+task: global_mmlu_full_fa_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_medical_genetics.yaml
new file mode 100644
index 000000000..c31679ec0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_fa_other_tasks
+task: global_mmlu_full_fa_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_miscellaneous.yaml
new file mode 100644
index 000000000..652d5a333
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_fa_other_tasks
+task: global_mmlu_full_fa_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_moral_disputes.yaml
new file mode 100644
index 000000000..16adcb262
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_fa_humanities_tasks
+task: global_mmlu_full_fa_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_moral_scenarios.yaml
new file mode 100644
index 000000000..92d018f25
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_fa_humanities_tasks
+task: global_mmlu_full_fa_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_nutrition.yaml
new file mode 100644
index 000000000..ae7e065ec
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_fa_other_tasks
+task: global_mmlu_full_fa_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_philosophy.yaml
new file mode 100644
index 000000000..cd8513da7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_fa_humanities_tasks
+task: global_mmlu_full_fa_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_prehistory.yaml
new file mode 100644
index 000000000..9fd6bb3d0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_fa_humanities_tasks
+task: global_mmlu_full_fa_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_accounting.yaml
new file mode 100644
index 000000000..99f6c3164
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_fa_other_tasks
+task: global_mmlu_full_fa_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_law.yaml
new file mode 100644
index 000000000..9fee460a4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_fa_humanities_tasks
+task: global_mmlu_full_fa_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_medicine.yaml
new file mode 100644
index 000000000..13d67d457
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_fa_other_tasks
+task: global_mmlu_full_fa_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_psychology.yaml
new file mode 100644
index 000000000..3e821145e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_fa_social_sciences_tasks
+task: global_mmlu_full_fa_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_public_relations.yaml
new file mode 100644
index 000000000..de6cc3118
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_fa_social_sciences_tasks
+task: global_mmlu_full_fa_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_security_studies.yaml
new file mode 100644
index 000000000..64d5fd14a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_fa_social_sciences_tasks
+task: global_mmlu_full_fa_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_sociology.yaml
new file mode 100644
index 000000000..cf3d9564b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_fa_social_sciences_tasks
+task: global_mmlu_full_fa_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_us_foreign_policy.yaml
new file mode 100644
index 000000000..38d51936c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_fa_social_sciences_tasks
+task: global_mmlu_full_fa_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_virology.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_virology.yaml
new file mode 100644
index 000000000..39c5188d3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_fa_other_tasks
+task: global_mmlu_full_fa_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_world_religions.yaml
new file mode 100644
index 000000000..44e6fc829
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fa_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_fa_humanities_tasks
+task: global_mmlu_full_fa_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/fa/utils.py b/lmms_eval/tasks/global_mmlu/full/fa/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fa/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/_fil_template_yaml b/lmms_eval/tasks/global_mmlu/full/fil/_fil_template_yaml
new file mode 100644
index 000000000..32dc097a1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/_fil_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: fil
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil.yaml b/lmms_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil.yaml
new file mode 100644
index 000000000..659c9d4b9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_fil
+task:
+  - global_mmlu_full_fil_stem
+  - global_mmlu_full_fil_other
+  - global_mmlu_full_fil_social_sciences
+  - global_mmlu_full_fil_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_humanities.yaml
new file mode 100644
index 000000000..061eb8189
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_fil_humanities
+task:
+  - global_mmlu_full_fil_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_other.yaml b/lmms_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_other.yaml
new file mode 100644
index 000000000..fea793eff
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_fil_other
+task:
+  - global_mmlu_full_fil_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_social_sciences.yaml
new file mode 100644
index 000000000..e9f793305
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_fil_social_sciences
+task:
+  - global_mmlu_full_fil_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_stem.yaml b/lmms_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_stem.yaml
new file mode 100644
index 000000000..2e567c70b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_fil_stem
+task:
+  - global_mmlu_full_fil_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_abstract_algebra.yaml
new file mode 100644
index 000000000..7eef19d6d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_fil_stem_tasks
+task: global_mmlu_full_fil_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_anatomy.yaml
new file mode 100644
index 000000000..e87d8d803
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_fil_stem_tasks
+task: global_mmlu_full_fil_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_astronomy.yaml
new file mode 100644
index 000000000..6c258877f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_fil_stem_tasks
+task: global_mmlu_full_fil_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_business_ethics.yaml
new file mode 100644
index 000000000..139f3cccf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_fil_other_tasks
+task: global_mmlu_full_fil_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_clinical_knowledge.yaml
new file mode 100644
index 000000000..fc160a99f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_fil_other_tasks
+task: global_mmlu_full_fil_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_biology.yaml
new file mode 100644
index 000000000..ff6fa3d84
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_fil_stem_tasks
+task: global_mmlu_full_fil_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_chemistry.yaml
new file mode 100644
index 000000000..61f0df508
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_fil_stem_tasks
+task: global_mmlu_full_fil_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_computer_science.yaml
new file mode 100644
index 000000000..1385b9345
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_fil_stem_tasks
+task: global_mmlu_full_fil_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_mathematics.yaml
new file mode 100644
index 000000000..afe15d7d5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_fil_stem_tasks
+task: global_mmlu_full_fil_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_medicine.yaml
new file mode 100644
index 000000000..221289f7d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_fil_other_tasks
+task: global_mmlu_full_fil_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_physics.yaml
new file mode 100644
index 000000000..863792b3c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_fil_stem_tasks
+task: global_mmlu_full_fil_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_computer_security.yaml
new file mode 100644
index 000000000..7971c6065
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_fil_stem_tasks
+task: global_mmlu_full_fil_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_conceptual_physics.yaml
new file mode 100644
index 000000000..77a75ccf0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_fil_stem_tasks
+task: global_mmlu_full_fil_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_econometrics.yaml
new file mode 100644
index 000000000..bd98fc8db
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_fil_social_sciences_tasks
+task: global_mmlu_full_fil_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_electrical_engineering.yaml
new file mode 100644
index 000000000..98e48a27c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_fil_stem_tasks
+task: global_mmlu_full_fil_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_elementary_mathematics.yaml
new file mode 100644
index 000000000..eba4149c2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_fil_stem_tasks
+task: global_mmlu_full_fil_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_formal_logic.yaml
new file mode 100644
index 000000000..f1796059f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_fil_humanities_tasks
+task: global_mmlu_full_fil_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_global_facts.yaml
new file mode 100644
index 000000000..96886181c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_fil_other_tasks
+task: global_mmlu_full_fil_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_biology.yaml
new file mode 100644
index 000000000..93d94120d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_fil_stem_tasks
+task: global_mmlu_full_fil_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_chemistry.yaml
new file mode 100644
index 000000000..9ec56d5be
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_fil_stem_tasks
+task: global_mmlu_full_fil_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_computer_science.yaml
new file mode 100644
index 000000000..82d86aed2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_fil_stem_tasks
+task: global_mmlu_full_fil_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_european_history.yaml
new file mode 100644
index 000000000..7fcdec0a1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_fil_humanities_tasks
+task: global_mmlu_full_fil_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_geography.yaml
new file mode 100644
index 000000000..962681922
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_fil_social_sciences_tasks
+task: global_mmlu_full_fil_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..f826de3da
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_fil_social_sciences_tasks
+task: global_mmlu_full_fil_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..104a70881
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_fil_social_sciences_tasks
+task: global_mmlu_full_fil_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_mathematics.yaml
new file mode 100644
index 000000000..1d499b4d9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_fil_stem_tasks
+task: global_mmlu_full_fil_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_microeconomics.yaml
new file mode 100644
index 000000000..43fcc04d9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_fil_social_sciences_tasks
+task: global_mmlu_full_fil_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_physics.yaml
new file mode 100644
index 000000000..175f31eef
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_fil_stem_tasks
+task: global_mmlu_full_fil_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_psychology.yaml
new file mode 100644
index 000000000..2fc2dd5ca
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_fil_social_sciences_tasks
+task: global_mmlu_full_fil_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_statistics.yaml
new file mode 100644
index 000000000..0540d57c8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_fil_stem_tasks
+task: global_mmlu_full_fil_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_us_history.yaml
new file mode 100644
index 000000000..d0801af2d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_fil_humanities_tasks
+task: global_mmlu_full_fil_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_world_history.yaml
new file mode 100644
index 000000000..724b7ce81
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_fil_humanities_tasks
+task: global_mmlu_full_fil_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_human_aging.yaml
new file mode 100644
index 000000000..6c2c11412
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_fil_other_tasks
+task: global_mmlu_full_fil_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_human_sexuality.yaml
new file mode 100644
index 000000000..1672d5b26
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_fil_social_sciences_tasks
+task: global_mmlu_full_fil_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_international_law.yaml
new file mode 100644
index 000000000..4c5da91c5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_fil_humanities_tasks
+task: global_mmlu_full_fil_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_jurisprudence.yaml
new file mode 100644
index 000000000..dea2b20b8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_fil_humanities_tasks
+task: global_mmlu_full_fil_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_logical_fallacies.yaml
new file mode 100644
index 000000000..6a30c7243
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_fil_humanities_tasks
+task: global_mmlu_full_fil_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_machine_learning.yaml
new file mode 100644
index 000000000..d2a7062c2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_fil_stem_tasks
+task: global_mmlu_full_fil_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_management.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_management.yaml
new file mode 100644
index 000000000..1ea568355
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_fil_other_tasks
+task: global_mmlu_full_fil_management
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_marketing.yaml
new file mode 100644
index 000000000..82d4490ad
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_fil_other_tasks
+task: global_mmlu_full_fil_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_medical_genetics.yaml
new file mode 100644
index 000000000..bdeb0984f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_fil_other_tasks
+task: global_mmlu_full_fil_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_miscellaneous.yaml
new file mode 100644
index 000000000..51c56a3be
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_fil_other_tasks
+task: global_mmlu_full_fil_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_moral_disputes.yaml
new file mode 100644
index 000000000..53148a542
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_fil_humanities_tasks
+task: global_mmlu_full_fil_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_moral_scenarios.yaml
new file mode 100644
index 000000000..fb5fecf2e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_fil_humanities_tasks
+task: global_mmlu_full_fil_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_nutrition.yaml
new file mode 100644
index 000000000..35859dc2d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_fil_other_tasks
+task: global_mmlu_full_fil_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_philosophy.yaml
new file mode 100644
index 000000000..dc2d414eb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_fil_humanities_tasks
+task: global_mmlu_full_fil_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_prehistory.yaml
new file mode 100644
index 000000000..abf65fd2f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_fil_humanities_tasks
+task: global_mmlu_full_fil_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_accounting.yaml
new file mode 100644
index 000000000..04ce34364
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_fil_other_tasks
+task: global_mmlu_full_fil_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_law.yaml
new file mode 100644
index 000000000..e5694cf45
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_fil_humanities_tasks
+task: global_mmlu_full_fil_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_medicine.yaml
new file mode 100644
index 000000000..e7fd0446f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_fil_other_tasks
+task: global_mmlu_full_fil_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_psychology.yaml
new file mode 100644
index 000000000..b9ce14aa0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_fil_social_sciences_tasks
+task: global_mmlu_full_fil_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_public_relations.yaml
new file mode 100644
index 000000000..fdae52986
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_fil_social_sciences_tasks
+task: global_mmlu_full_fil_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_security_studies.yaml
new file mode 100644
index 000000000..4a03eec19
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_fil_social_sciences_tasks
+task: global_mmlu_full_fil_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_sociology.yaml
new file mode 100644
index 000000000..bc0ed0523
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_fil_social_sciences_tasks
+task: global_mmlu_full_fil_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_us_foreign_policy.yaml
new file mode 100644
index 000000000..ed40afb61
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_fil_social_sciences_tasks
+task: global_mmlu_full_fil_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_virology.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_virology.yaml
new file mode 100644
index 000000000..85ed4d427
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_fil_other_tasks
+task: global_mmlu_full_fil_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_world_religions.yaml
new file mode 100644
index 000000000..3ee6bce18
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fil_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_fil_humanities_tasks
+task: global_mmlu_full_fil_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/fil/utils.py b/lmms_eval/tasks/global_mmlu/full/fil/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fil/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/_fr_template_yaml b/lmms_eval/tasks/global_mmlu/full/fr/_fr_template_yaml
new file mode 100644
index 000000000..47ca79725
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/_fr_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: fr
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr.yaml b/lmms_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr.yaml
new file mode 100644
index 000000000..7857a1932
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_fr
+task:
+  - global_mmlu_full_fr_stem
+  - global_mmlu_full_fr_other
+  - global_mmlu_full_fr_social_sciences
+  - global_mmlu_full_fr_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_humanities.yaml
new file mode 100644
index 000000000..697e3a29e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_fr_humanities
+task:
+  - global_mmlu_full_fr_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_other.yaml b/lmms_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_other.yaml
new file mode 100644
index 000000000..9b2ada6b3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_fr_other
+task:
+  - global_mmlu_full_fr_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_social_sciences.yaml
new file mode 100644
index 000000000..ac7e46056
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_fr_social_sciences
+task:
+  - global_mmlu_full_fr_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_stem.yaml b/lmms_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_stem.yaml
new file mode 100644
index 000000000..c81d601f3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_fr_stem
+task:
+  - global_mmlu_full_fr_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_abstract_algebra.yaml
new file mode 100644
index 000000000..bf7d76c38
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_fr_stem_tasks
+task: global_mmlu_full_fr_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_anatomy.yaml
new file mode 100644
index 000000000..e9a96927c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_fr_stem_tasks
+task: global_mmlu_full_fr_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_astronomy.yaml
new file mode 100644
index 000000000..6e4ca5a76
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_fr_stem_tasks
+task: global_mmlu_full_fr_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_business_ethics.yaml
new file mode 100644
index 000000000..df3c1fbd6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_fr_other_tasks
+task: global_mmlu_full_fr_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_clinical_knowledge.yaml
new file mode 100644
index 000000000..b0daa2e6e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_fr_other_tasks
+task: global_mmlu_full_fr_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_biology.yaml
new file mode 100644
index 000000000..1e9975788
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_fr_stem_tasks
+task: global_mmlu_full_fr_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_chemistry.yaml
new file mode 100644
index 000000000..9c1c31897
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_fr_stem_tasks
+task: global_mmlu_full_fr_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_computer_science.yaml
new file mode 100644
index 000000000..078108f8c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_fr_stem_tasks
+task: global_mmlu_full_fr_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_mathematics.yaml
new file mode 100644
index 000000000..bf2f2940e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_fr_stem_tasks
+task: global_mmlu_full_fr_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_medicine.yaml
new file mode 100644
index 000000000..8c9ccc80e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_fr_other_tasks
+task: global_mmlu_full_fr_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_physics.yaml
new file mode 100644
index 000000000..01dcea370
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_fr_stem_tasks
+task: global_mmlu_full_fr_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_computer_security.yaml
new file mode 100644
index 000000000..794f64be1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_fr_stem_tasks
+task: global_mmlu_full_fr_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_conceptual_physics.yaml
new file mode 100644
index 000000000..12c6afc2a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_fr_stem_tasks
+task: global_mmlu_full_fr_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_econometrics.yaml
new file mode 100644
index 000000000..1f33ddaba
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_fr_social_sciences_tasks
+task: global_mmlu_full_fr_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_electrical_engineering.yaml
new file mode 100644
index 000000000..8dcb05856
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_fr_stem_tasks
+task: global_mmlu_full_fr_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_elementary_mathematics.yaml
new file mode 100644
index 000000000..2658ce96a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_fr_stem_tasks
+task: global_mmlu_full_fr_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_formal_logic.yaml
new file mode 100644
index 000000000..5239cb1c1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_fr_humanities_tasks
+task: global_mmlu_full_fr_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_global_facts.yaml
new file mode 100644
index 000000000..2763dcb53
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_fr_other_tasks
+task: global_mmlu_full_fr_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_biology.yaml
new file mode 100644
index 000000000..2a6a26c9b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_fr_stem_tasks
+task: global_mmlu_full_fr_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_chemistry.yaml
new file mode 100644
index 000000000..6ffacc298
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_fr_stem_tasks
+task: global_mmlu_full_fr_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_computer_science.yaml
new file mode 100644
index 000000000..d17204224
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_fr_stem_tasks
+task: global_mmlu_full_fr_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_european_history.yaml
new file mode 100644
index 000000000..9788e7be7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_fr_humanities_tasks
+task: global_mmlu_full_fr_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_geography.yaml
new file mode 100644
index 000000000..3e2ff22ea
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_fr_social_sciences_tasks
+task: global_mmlu_full_fr_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..2aba3b612
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_fr_social_sciences_tasks
+task: global_mmlu_full_fr_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..21fb1df5b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_fr_social_sciences_tasks
+task: global_mmlu_full_fr_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_mathematics.yaml
new file mode 100644
index 000000000..a975d1fcc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_fr_stem_tasks
+task: global_mmlu_full_fr_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_microeconomics.yaml
new file mode 100644
index 000000000..ff654ff35
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_fr_social_sciences_tasks
+task: global_mmlu_full_fr_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_physics.yaml
new file mode 100644
index 000000000..4038c956d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_fr_stem_tasks
+task: global_mmlu_full_fr_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_psychology.yaml
new file mode 100644
index 000000000..a65da7802
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_fr_social_sciences_tasks
+task: global_mmlu_full_fr_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_statistics.yaml
new file mode 100644
index 000000000..37c75136b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_fr_stem_tasks
+task: global_mmlu_full_fr_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_us_history.yaml
new file mode 100644
index 000000000..a0e123f26
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_fr_humanities_tasks
+task: global_mmlu_full_fr_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_world_history.yaml
new file mode 100644
index 000000000..e2a9cf6b1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_fr_humanities_tasks
+task: global_mmlu_full_fr_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_human_aging.yaml
new file mode 100644
index 000000000..b9e9ece96
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_fr_other_tasks
+task: global_mmlu_full_fr_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_human_sexuality.yaml
new file mode 100644
index 000000000..eac30d27f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_fr_social_sciences_tasks
+task: global_mmlu_full_fr_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_international_law.yaml
new file mode 100644
index 000000000..2e15b0fb1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_fr_humanities_tasks
+task: global_mmlu_full_fr_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_jurisprudence.yaml
new file mode 100644
index 000000000..f42079c92
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_fr_humanities_tasks
+task: global_mmlu_full_fr_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_logical_fallacies.yaml
new file mode 100644
index 000000000..68ebdb717
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_fr_humanities_tasks
+task: global_mmlu_full_fr_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_machine_learning.yaml
new file mode 100644
index 000000000..25a8df3a8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_fr_stem_tasks
+task: global_mmlu_full_fr_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_management.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_management.yaml
new file mode 100644
index 000000000..73f7d8692
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_fr_other_tasks
+task: global_mmlu_full_fr_management
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_marketing.yaml
new file mode 100644
index 000000000..8a19b83e9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_fr_other_tasks
+task: global_mmlu_full_fr_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_medical_genetics.yaml
new file mode 100644
index 000000000..d15774f13
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_fr_other_tasks
+task: global_mmlu_full_fr_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_miscellaneous.yaml
new file mode 100644
index 000000000..d75197097
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_fr_other_tasks
+task: global_mmlu_full_fr_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_moral_disputes.yaml
new file mode 100644
index 000000000..f625921e5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_fr_humanities_tasks
+task: global_mmlu_full_fr_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_moral_scenarios.yaml
new file mode 100644
index 000000000..4575ca044
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_fr_humanities_tasks
+task: global_mmlu_full_fr_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_nutrition.yaml
new file mode 100644
index 000000000..61521c91d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_fr_other_tasks
+task: global_mmlu_full_fr_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_philosophy.yaml
new file mode 100644
index 000000000..7f771c342
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_fr_humanities_tasks
+task: global_mmlu_full_fr_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_prehistory.yaml
new file mode 100644
index 000000000..3bcac0f5b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_fr_humanities_tasks
+task: global_mmlu_full_fr_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_accounting.yaml
new file mode 100644
index 000000000..a06a7af51
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_fr_other_tasks
+task: global_mmlu_full_fr_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_law.yaml
new file mode 100644
index 000000000..2ecf2e8e7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_fr_humanities_tasks
+task: global_mmlu_full_fr_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_medicine.yaml
new file mode 100644
index 000000000..983a2d380
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_fr_other_tasks
+task: global_mmlu_full_fr_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_psychology.yaml
new file mode 100644
index 000000000..59d9aa309
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_fr_social_sciences_tasks
+task: global_mmlu_full_fr_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_public_relations.yaml
new file mode 100644
index 000000000..d84b7ad07
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_fr_social_sciences_tasks
+task: global_mmlu_full_fr_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_security_studies.yaml
new file mode 100644
index 000000000..fcd82b7d8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_fr_social_sciences_tasks
+task: global_mmlu_full_fr_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_sociology.yaml
new file mode 100644
index 000000000..ff7b8fd31
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_fr_social_sciences_tasks
+task: global_mmlu_full_fr_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_us_foreign_policy.yaml
new file mode 100644
index 000000000..d92c20955
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_fr_social_sciences_tasks
+task: global_mmlu_full_fr_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_virology.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_virology.yaml
new file mode 100644
index 000000000..211c96a79
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_fr_other_tasks
+task: global_mmlu_full_fr_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_world_religions.yaml
new file mode 100644
index 000000000..f1f168aed
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _fr_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_fr_humanities_tasks
+task: global_mmlu_full_fr_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/fr/utils.py b/lmms_eval/tasks/global_mmlu/full/fr/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/fr/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha.yaml b/lmms_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha.yaml
new file mode 100644
index 000000000..a50084174
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_ha
+task:
+  - global_mmlu_full_ha_stem
+  - global_mmlu_full_ha_other
+  - global_mmlu_full_ha_social_sciences
+  - global_mmlu_full_ha_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_humanities.yaml
new file mode 100644
index 000000000..84cce38d7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ha_humanities
+task:
+  - global_mmlu_full_ha_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_other.yaml b/lmms_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_other.yaml
new file mode 100644
index 000000000..73a6ea0de
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ha_other
+task:
+  - global_mmlu_full_ha_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_social_sciences.yaml
new file mode 100644
index 000000000..8b520a5dc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ha_social_sciences
+task:
+  - global_mmlu_full_ha_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_stem.yaml b/lmms_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_stem.yaml
new file mode 100644
index 000000000..6213d280b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ha_stem
+task:
+  - global_mmlu_full_ha_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/_ha_template_yaml b/lmms_eval/tasks/global_mmlu/full/ha/_ha_template_yaml
new file mode 100644
index 000000000..8521fe50d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/_ha_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: ha
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_abstract_algebra.yaml
new file mode 100644
index 000000000..62ad5e7d7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_anatomy.yaml
new file mode 100644
index 000000000..2ead0f6c1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_astronomy.yaml
new file mode 100644
index 000000000..1616398f8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_business_ethics.yaml
new file mode 100644
index 000000000..c1719b0e0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_ha_other_tasks
+task: global_mmlu_full_ha_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_clinical_knowledge.yaml
new file mode 100644
index 000000000..dcef5e272
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_ha_other_tasks
+task: global_mmlu_full_ha_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_biology.yaml
new file mode 100644
index 000000000..f28256946
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_chemistry.yaml
new file mode 100644
index 000000000..73a422e3d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_computer_science.yaml
new file mode 100644
index 000000000..7bdb65c9e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_mathematics.yaml
new file mode 100644
index 000000000..47e5326ce
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_medicine.yaml
new file mode 100644
index 000000000..9065f0855
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_ha_other_tasks
+task: global_mmlu_full_ha_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_physics.yaml
new file mode 100644
index 000000000..40aa11c5d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_computer_security.yaml
new file mode 100644
index 000000000..38d1e9c25
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_conceptual_physics.yaml
new file mode 100644
index 000000000..7326514a4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_econometrics.yaml
new file mode 100644
index 000000000..e865b6bc6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_ha_social_sciences_tasks
+task: global_mmlu_full_ha_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_electrical_engineering.yaml
new file mode 100644
index 000000000..9457d1bb9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_elementary_mathematics.yaml
new file mode 100644
index 000000000..e04fb1ff6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_formal_logic.yaml
new file mode 100644
index 000000000..03c9cbac0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_ha_humanities_tasks
+task: global_mmlu_full_ha_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_global_facts.yaml
new file mode 100644
index 000000000..db104be57
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_ha_other_tasks
+task: global_mmlu_full_ha_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_biology.yaml
new file mode 100644
index 000000000..729fed2be
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_chemistry.yaml
new file mode 100644
index 000000000..13f5621b2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_computer_science.yaml
new file mode 100644
index 000000000..1914e1fb8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_european_history.yaml
new file mode 100644
index 000000000..fa878b037
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_ha_humanities_tasks
+task: global_mmlu_full_ha_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_geography.yaml
new file mode 100644
index 000000000..10a136746
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_ha_social_sciences_tasks
+task: global_mmlu_full_ha_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..eebac4096
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_ha_social_sciences_tasks
+task: global_mmlu_full_ha_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..0a22ab842
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_ha_social_sciences_tasks
+task: global_mmlu_full_ha_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_mathematics.yaml
new file mode 100644
index 000000000..fc681f909
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_microeconomics.yaml
new file mode 100644
index 000000000..81bb343c3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_ha_social_sciences_tasks
+task: global_mmlu_full_ha_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_physics.yaml
new file mode 100644
index 000000000..2bc4cc4d9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_psychology.yaml
new file mode 100644
index 000000000..c5d46e5ec
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_ha_social_sciences_tasks
+task: global_mmlu_full_ha_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_statistics.yaml
new file mode 100644
index 000000000..4848cc314
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_us_history.yaml
new file mode 100644
index 000000000..7a22c79ab
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_ha_humanities_tasks
+task: global_mmlu_full_ha_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_world_history.yaml
new file mode 100644
index 000000000..138822798
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_ha_humanities_tasks
+task: global_mmlu_full_ha_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_human_aging.yaml
new file mode 100644
index 000000000..51ff436ba
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_ha_other_tasks
+task: global_mmlu_full_ha_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_human_sexuality.yaml
new file mode 100644
index 000000000..1a36fb865
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_ha_social_sciences_tasks
+task: global_mmlu_full_ha_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_international_law.yaml
new file mode 100644
index 000000000..f1c9cc1c4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_ha_humanities_tasks
+task: global_mmlu_full_ha_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_jurisprudence.yaml
new file mode 100644
index 000000000..4bc1314ba
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_ha_humanities_tasks
+task: global_mmlu_full_ha_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_logical_fallacies.yaml
new file mode 100644
index 000000000..259534b9a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_ha_humanities_tasks
+task: global_mmlu_full_ha_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_machine_learning.yaml
new file mode 100644
index 000000000..c94a073b9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_ha_stem_tasks
+task: global_mmlu_full_ha_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_management.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_management.yaml
new file mode 100644
index 000000000..666d47202
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_ha_other_tasks
+task: global_mmlu_full_ha_management
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_marketing.yaml
new file mode 100644
index 000000000..9528a1f37
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_ha_other_tasks
+task: global_mmlu_full_ha_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_medical_genetics.yaml
new file mode 100644
index 000000000..92f0a408d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_ha_other_tasks
+task: global_mmlu_full_ha_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_miscellaneous.yaml
new file mode 100644
index 000000000..fc97a8dc9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_ha_other_tasks
+task: global_mmlu_full_ha_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_moral_disputes.yaml
new file mode 100644
index 000000000..dbcf96c22
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_ha_humanities_tasks
+task: global_mmlu_full_ha_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_moral_scenarios.yaml
new file mode 100644
index 000000000..aa7b42669
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_ha_humanities_tasks
+task: global_mmlu_full_ha_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_nutrition.yaml
new file mode 100644
index 000000000..b413e4be1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_ha_other_tasks
+task: global_mmlu_full_ha_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_philosophy.yaml
new file mode 100644
index 000000000..118e48014
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_ha_humanities_tasks
+task: global_mmlu_full_ha_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_prehistory.yaml
new file mode 100644
index 000000000..a310d0230
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_ha_humanities_tasks
+task: global_mmlu_full_ha_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_accounting.yaml
new file mode 100644
index 000000000..79536ddc6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_ha_other_tasks
+task: global_mmlu_full_ha_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_law.yaml
new file mode 100644
index 000000000..613170da1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_ha_humanities_tasks
+task: global_mmlu_full_ha_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_medicine.yaml
new file mode 100644
index 000000000..bd65c2330
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_ha_other_tasks
+task: global_mmlu_full_ha_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_psychology.yaml
new file mode 100644
index 000000000..cf7ecb1f5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_ha_social_sciences_tasks
+task: global_mmlu_full_ha_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_public_relations.yaml
new file mode 100644
index 000000000..c9cba53f9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_ha_social_sciences_tasks
+task: global_mmlu_full_ha_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_security_studies.yaml
new file mode 100644
index 000000000..fe7676860
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_ha_social_sciences_tasks
+task: global_mmlu_full_ha_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_sociology.yaml
new file mode 100644
index 000000000..94f8e311f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_ha_social_sciences_tasks
+task: global_mmlu_full_ha_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_us_foreign_policy.yaml
new file mode 100644
index 000000000..54f82b3f3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_ha_social_sciences_tasks
+task: global_mmlu_full_ha_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_virology.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_virology.yaml
new file mode 100644
index 000000000..ce7d224d5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_ha_other_tasks
+task: global_mmlu_full_ha_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_world_religions.yaml
new file mode 100644
index 000000000..67a6d33db
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ha_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_ha_humanities_tasks
+task: global_mmlu_full_ha_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/ha/utils.py b/lmms_eval/tasks/global_mmlu/full/ha/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ha/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he.yaml b/lmms_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he.yaml
new file mode 100644
index 000000000..4952391a2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_he
+task:
+  - global_mmlu_full_he_stem
+  - global_mmlu_full_he_other
+  - global_mmlu_full_he_social_sciences
+  - global_mmlu_full_he_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_humanities.yaml
new file mode 100644
index 000000000..678ee0d4c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_he_humanities
+task:
+  - global_mmlu_full_he_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_other.yaml b/lmms_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_other.yaml
new file mode 100644
index 000000000..c99b4806a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_he_other
+task:
+  - global_mmlu_full_he_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_social_sciences.yaml
new file mode 100644
index 000000000..129068958
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_he_social_sciences
+task:
+  - global_mmlu_full_he_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_stem.yaml b/lmms_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_stem.yaml
new file mode 100644
index 000000000..f6e76e7a8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_he_stem
+task:
+  - global_mmlu_full_he_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/he/_he_template_yaml b/lmms_eval/tasks/global_mmlu/full/he/_he_template_yaml
new file mode 100644
index 000000000..b6ec9fc82
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/_he_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: he
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_abstract_algebra.yaml
new file mode 100644
index 000000000..fb197c3de
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_he_stem_tasks
+task: global_mmlu_full_he_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_anatomy.yaml
new file mode 100644
index 000000000..3ab9ee20e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_he_stem_tasks
+task: global_mmlu_full_he_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_astronomy.yaml
new file mode 100644
index 000000000..8950b1e03
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_he_stem_tasks
+task: global_mmlu_full_he_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_business_ethics.yaml
new file mode 100644
index 000000000..8c114348b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_he_other_tasks
+task: global_mmlu_full_he_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_clinical_knowledge.yaml
new file mode 100644
index 000000000..1324a04dd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_he_other_tasks
+task: global_mmlu_full_he_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_biology.yaml
new file mode 100644
index 000000000..cecddc603
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_he_stem_tasks
+task: global_mmlu_full_he_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_chemistry.yaml
new file mode 100644
index 000000000..2c0f8b5d1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_he_stem_tasks
+task: global_mmlu_full_he_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_computer_science.yaml
new file mode 100644
index 000000000..b4c36a414
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_he_stem_tasks
+task: global_mmlu_full_he_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_mathematics.yaml
new file mode 100644
index 000000000..3633d5374
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_he_stem_tasks
+task: global_mmlu_full_he_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_medicine.yaml
new file mode 100644
index 000000000..a28c592e5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_he_other_tasks
+task: global_mmlu_full_he_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_physics.yaml
new file mode 100644
index 000000000..3893b9aa0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_he_stem_tasks
+task: global_mmlu_full_he_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_computer_security.yaml
new file mode 100644
index 000000000..4167874ea
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_he_stem_tasks
+task: global_mmlu_full_he_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_conceptual_physics.yaml
new file mode 100644
index 000000000..2ee92851a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_he_stem_tasks
+task: global_mmlu_full_he_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_econometrics.yaml
new file mode 100644
index 000000000..9ceb32776
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_he_social_sciences_tasks
+task: global_mmlu_full_he_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_electrical_engineering.yaml
new file mode 100644
index 000000000..00658e28e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_he_stem_tasks
+task: global_mmlu_full_he_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_elementary_mathematics.yaml
new file mode 100644
index 000000000..10f2ac18d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_he_stem_tasks
+task: global_mmlu_full_he_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_formal_logic.yaml
new file mode 100644
index 000000000..aac3f8dab
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_he_humanities_tasks
+task: global_mmlu_full_he_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_global_facts.yaml
new file mode 100644
index 000000000..299a73efd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_he_other_tasks
+task: global_mmlu_full_he_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_biology.yaml
new file mode 100644
index 000000000..9d3ba8935
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_he_stem_tasks
+task: global_mmlu_full_he_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_chemistry.yaml
new file mode 100644
index 000000000..f67f8ef3d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_he_stem_tasks
+task: global_mmlu_full_he_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_computer_science.yaml
new file mode 100644
index 000000000..7ca8b6f86
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_he_stem_tasks
+task: global_mmlu_full_he_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_european_history.yaml
new file mode 100644
index 000000000..58e4081a5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_he_humanities_tasks
+task: global_mmlu_full_he_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_geography.yaml
new file mode 100644
index 000000000..2d76e3879
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_he_social_sciences_tasks
+task: global_mmlu_full_he_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..e37451104
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_he_social_sciences_tasks
+task: global_mmlu_full_he_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..fa0b7c711
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_he_social_sciences_tasks
+task: global_mmlu_full_he_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_mathematics.yaml
new file mode 100644
index 000000000..7f78a5c59
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_he_stem_tasks
+task: global_mmlu_full_he_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_microeconomics.yaml
new file mode 100644
index 000000000..15be92433
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_he_social_sciences_tasks
+task: global_mmlu_full_he_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_physics.yaml
new file mode 100644
index 000000000..6f309c0bd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_he_stem_tasks
+task: global_mmlu_full_he_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_psychology.yaml
new file mode 100644
index 000000000..1ae831c60
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_he_social_sciences_tasks
+task: global_mmlu_full_he_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_statistics.yaml
new file mode 100644
index 000000000..3a2e81704
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_he_stem_tasks
+task: global_mmlu_full_he_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_us_history.yaml
new file mode 100644
index 000000000..c05da45aa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_he_humanities_tasks
+task: global_mmlu_full_he_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_world_history.yaml
new file mode 100644
index 000000000..b818e4fec
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_he_humanities_tasks
+task: global_mmlu_full_he_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_human_aging.yaml
new file mode 100644
index 000000000..49f7ce5da
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_he_other_tasks
+task: global_mmlu_full_he_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_human_sexuality.yaml
new file mode 100644
index 000000000..91d085670
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_he_social_sciences_tasks
+task: global_mmlu_full_he_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_international_law.yaml
new file mode 100644
index 000000000..1bedb4f42
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_he_humanities_tasks
+task: global_mmlu_full_he_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_jurisprudence.yaml
new file mode 100644
index 000000000..39fe15a2b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_he_humanities_tasks
+task: global_mmlu_full_he_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_logical_fallacies.yaml
new file mode 100644
index 000000000..e54b58b33
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_he_humanities_tasks
+task: global_mmlu_full_he_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_machine_learning.yaml
new file mode 100644
index 000000000..8190e96ac
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_he_stem_tasks
+task: global_mmlu_full_he_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_management.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_management.yaml
new file mode 100644
index 000000000..d5811f804
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_he_other_tasks
+task: global_mmlu_full_he_management
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_marketing.yaml
new file mode 100644
index 000000000..7fe44232b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_he_other_tasks
+task: global_mmlu_full_he_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_medical_genetics.yaml
new file mode 100644
index 000000000..8c9082c5f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_he_other_tasks
+task: global_mmlu_full_he_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_miscellaneous.yaml
new file mode 100644
index 000000000..bc419deea
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_he_other_tasks
+task: global_mmlu_full_he_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_moral_disputes.yaml
new file mode 100644
index 000000000..d889642be
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_he_humanities_tasks
+task: global_mmlu_full_he_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_moral_scenarios.yaml
new file mode 100644
index 000000000..115548234
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_he_humanities_tasks
+task: global_mmlu_full_he_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_nutrition.yaml
new file mode 100644
index 000000000..30d49701b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_he_other_tasks
+task: global_mmlu_full_he_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_philosophy.yaml
new file mode 100644
index 000000000..458632de4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_he_humanities_tasks
+task: global_mmlu_full_he_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_prehistory.yaml
new file mode 100644
index 000000000..938356731
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_he_humanities_tasks
+task: global_mmlu_full_he_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_accounting.yaml
new file mode 100644
index 000000000..aed286368
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_he_other_tasks
+task: global_mmlu_full_he_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_law.yaml
new file mode 100644
index 000000000..38a9e3cc2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_he_humanities_tasks
+task: global_mmlu_full_he_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_medicine.yaml
new file mode 100644
index 000000000..e8ca950c5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_he_other_tasks
+task: global_mmlu_full_he_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_psychology.yaml
new file mode 100644
index 000000000..f82c28923
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_he_social_sciences_tasks
+task: global_mmlu_full_he_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_public_relations.yaml
new file mode 100644
index 000000000..e3aff6615
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_he_social_sciences_tasks
+task: global_mmlu_full_he_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_security_studies.yaml
new file mode 100644
index 000000000..e99aa0154
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_he_social_sciences_tasks
+task: global_mmlu_full_he_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_sociology.yaml
new file mode 100644
index 000000000..de81b92c9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_he_social_sciences_tasks
+task: global_mmlu_full_he_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_us_foreign_policy.yaml
new file mode 100644
index 000000000..7be650440
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_he_social_sciences_tasks
+task: global_mmlu_full_he_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_virology.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_virology.yaml
new file mode 100644
index 000000000..b6f51e1b9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_he_other_tasks
+task: global_mmlu_full_he_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_world_religions.yaml
new file mode 100644
index 000000000..e3d10a0d5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _he_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_he_humanities_tasks
+task: global_mmlu_full_he_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/he/utils.py b/lmms_eval/tasks/global_mmlu/full/he/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/he/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi.yaml b/lmms_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi.yaml
new file mode 100644
index 000000000..c899be84f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_hi
+task:
+  - global_mmlu_full_hi_stem
+  - global_mmlu_full_hi_other
+  - global_mmlu_full_hi_social_sciences
+  - global_mmlu_full_hi_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_humanities.yaml
new file mode 100644
index 000000000..36492fa3f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_hi_humanities
+task:
+  - global_mmlu_full_hi_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_other.yaml b/lmms_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_other.yaml
new file mode 100644
index 000000000..08dc16b0f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_hi_other
+task:
+  - global_mmlu_full_hi_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_social_sciences.yaml
new file mode 100644
index 000000000..0a4dfdd72
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_hi_social_sciences
+task:
+  - global_mmlu_full_hi_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_stem.yaml b/lmms_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_stem.yaml
new file mode 100644
index 000000000..7a0123ae4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_hi_stem
+task:
+  - global_mmlu_full_hi_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/_hi_template_yaml b/lmms_eval/tasks/global_mmlu/full/hi/_hi_template_yaml
new file mode 100644
index 000000000..18c6286eb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/_hi_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: hi
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_abstract_algebra.yaml
new file mode 100644
index 000000000..f239f067d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_hi_stem_tasks
+task: global_mmlu_full_hi_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_anatomy.yaml
new file mode 100644
index 000000000..dfcd776ed
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_hi_stem_tasks
+task: global_mmlu_full_hi_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_astronomy.yaml
new file mode 100644
index 000000000..dbb6763d3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_hi_stem_tasks
+task: global_mmlu_full_hi_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_business_ethics.yaml
new file mode 100644
index 000000000..5882427e7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_hi_other_tasks
+task: global_mmlu_full_hi_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_clinical_knowledge.yaml
new file mode 100644
index 000000000..7b0c6c3b0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_hi_other_tasks
+task: global_mmlu_full_hi_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_biology.yaml
new file mode 100644
index 000000000..d5326c8df
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_hi_stem_tasks
+task: global_mmlu_full_hi_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_chemistry.yaml
new file mode 100644
index 000000000..bf9e21300
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_hi_stem_tasks
+task: global_mmlu_full_hi_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_computer_science.yaml
new file mode 100644
index 000000000..c79f4250c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_hi_stem_tasks
+task: global_mmlu_full_hi_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_mathematics.yaml
new file mode 100644
index 000000000..4e8b0427b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_hi_stem_tasks
+task: global_mmlu_full_hi_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_medicine.yaml
new file mode 100644
index 000000000..7e8c0df2c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_hi_other_tasks
+task: global_mmlu_full_hi_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_physics.yaml
new file mode 100644
index 000000000..5fe337ee1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_hi_stem_tasks
+task: global_mmlu_full_hi_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_computer_security.yaml
new file mode 100644
index 000000000..029a02e07
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_hi_stem_tasks
+task: global_mmlu_full_hi_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_conceptual_physics.yaml
new file mode 100644
index 000000000..a67489742
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_hi_stem_tasks
+task: global_mmlu_full_hi_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_econometrics.yaml
new file mode 100644
index 000000000..355053b25
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_hi_social_sciences_tasks
+task: global_mmlu_full_hi_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_electrical_engineering.yaml
new file mode 100644
index 000000000..04dca10d7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_hi_stem_tasks
+task: global_mmlu_full_hi_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_elementary_mathematics.yaml
new file mode 100644
index 000000000..ca7a30832
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_hi_stem_tasks
+task: global_mmlu_full_hi_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_formal_logic.yaml
new file mode 100644
index 000000000..ae534fa62
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_hi_humanities_tasks
+task: global_mmlu_full_hi_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_global_facts.yaml
new file mode 100644
index 000000000..096fd58bc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_hi_other_tasks
+task: global_mmlu_full_hi_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_biology.yaml
new file mode 100644
index 000000000..9ef04ee57
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_hi_stem_tasks
+task: global_mmlu_full_hi_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_chemistry.yaml
new file mode 100644
index 000000000..4e8913e52
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_hi_stem_tasks
+task: global_mmlu_full_hi_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_computer_science.yaml
new file mode 100644
index 000000000..180eef756
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_hi_stem_tasks
+task: global_mmlu_full_hi_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_european_history.yaml
new file mode 100644
index 000000000..32abd63b6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_hi_humanities_tasks
+task: global_mmlu_full_hi_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_geography.yaml
new file mode 100644
index 000000000..1089908b0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_hi_social_sciences_tasks
+task: global_mmlu_full_hi_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..fb22bb510
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_hi_social_sciences_tasks
+task: global_mmlu_full_hi_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..affc27c04
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_hi_social_sciences_tasks
+task: global_mmlu_full_hi_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_mathematics.yaml
new file mode 100644
index 000000000..59f97c949
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_hi_stem_tasks
+task: global_mmlu_full_hi_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_microeconomics.yaml
new file mode 100644
index 000000000..a7506a4cf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_hi_social_sciences_tasks
+task: global_mmlu_full_hi_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_physics.yaml
new file mode 100644
index 000000000..406035bd4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_hi_stem_tasks
+task: global_mmlu_full_hi_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_psychology.yaml
new file mode 100644
index 000000000..f5c2be371
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_hi_social_sciences_tasks
+task: global_mmlu_full_hi_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_statistics.yaml
new file mode 100644
index 000000000..a955febe3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_hi_stem_tasks
+task: global_mmlu_full_hi_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_us_history.yaml
new file mode 100644
index 000000000..6a5573f8b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_hi_humanities_tasks
+task: global_mmlu_full_hi_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_world_history.yaml
new file mode 100644
index 000000000..38ce46800
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_hi_humanities_tasks
+task: global_mmlu_full_hi_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_human_aging.yaml
new file mode 100644
index 000000000..2486301f8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_hi_other_tasks
+task: global_mmlu_full_hi_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_human_sexuality.yaml
new file mode 100644
index 000000000..8f889885b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_hi_social_sciences_tasks
+task: global_mmlu_full_hi_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_international_law.yaml
new file mode 100644
index 000000000..2cb0d834e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_hi_humanities_tasks
+task: global_mmlu_full_hi_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_jurisprudence.yaml
new file mode 100644
index 000000000..113291306
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_hi_humanities_tasks
+task: global_mmlu_full_hi_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_logical_fallacies.yaml
new file mode 100644
index 000000000..e22cedbe2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_hi_humanities_tasks
+task: global_mmlu_full_hi_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_machine_learning.yaml
new file mode 100644
index 000000000..134ab0806
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_hi_stem_tasks
+task: global_mmlu_full_hi_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_management.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_management.yaml
new file mode 100644
index 000000000..e523b5d1a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_hi_other_tasks
+task: global_mmlu_full_hi_management
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_marketing.yaml
new file mode 100644
index 000000000..11d8930b7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_hi_other_tasks
+task: global_mmlu_full_hi_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_medical_genetics.yaml
new file mode 100644
index 000000000..ad38e3e62
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_hi_other_tasks
+task: global_mmlu_full_hi_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_miscellaneous.yaml
new file mode 100644
index 000000000..c31f88832
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_hi_other_tasks
+task: global_mmlu_full_hi_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_moral_disputes.yaml
new file mode 100644
index 000000000..01145f6f7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_hi_humanities_tasks
+task: global_mmlu_full_hi_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_moral_scenarios.yaml
new file mode 100644
index 000000000..4acbb127b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_hi_humanities_tasks
+task: global_mmlu_full_hi_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_nutrition.yaml
new file mode 100644
index 000000000..889c00182
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_hi_other_tasks
+task: global_mmlu_full_hi_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_philosophy.yaml
new file mode 100644
index 000000000..2a8aeb4d3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_hi_humanities_tasks
+task: global_mmlu_full_hi_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_prehistory.yaml
new file mode 100644
index 000000000..ad80a3c08
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_hi_humanities_tasks
+task: global_mmlu_full_hi_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_accounting.yaml
new file mode 100644
index 000000000..1f5477899
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_hi_other_tasks
+task: global_mmlu_full_hi_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_law.yaml
new file mode 100644
index 000000000..836d577d3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_hi_humanities_tasks
+task: global_mmlu_full_hi_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_medicine.yaml
new file mode 100644
index 000000000..7a8e7db92
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_hi_other_tasks
+task: global_mmlu_full_hi_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_psychology.yaml
new file mode 100644
index 000000000..b4ebc1a5f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_hi_social_sciences_tasks
+task: global_mmlu_full_hi_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_public_relations.yaml
new file mode 100644
index 000000000..7bbf959cc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_hi_social_sciences_tasks
+task: global_mmlu_full_hi_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_security_studies.yaml
new file mode 100644
index 000000000..7faa9d43d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_hi_social_sciences_tasks
+task: global_mmlu_full_hi_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_sociology.yaml
new file mode 100644
index 000000000..b0ca49ae2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_hi_social_sciences_tasks
+task: global_mmlu_full_hi_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_us_foreign_policy.yaml
new file mode 100644
index 000000000..d5fd9f0ff
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_hi_social_sciences_tasks
+task: global_mmlu_full_hi_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_virology.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_virology.yaml
new file mode 100644
index 000000000..843ea2540
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_hi_other_tasks
+task: global_mmlu_full_hi_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_world_religions.yaml
new file mode 100644
index 000000000..f5e56ce04
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_hi_humanities_tasks
+task: global_mmlu_full_hi_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/hi/utils.py b/lmms_eval/tasks/global_mmlu/full/hi/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/hi/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id.yaml b/lmms_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id.yaml
new file mode 100644
index 000000000..fd29a2ad1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_id
+task:
+  - global_mmlu_full_id_stem
+  - global_mmlu_full_id_other
+  - global_mmlu_full_id_social_sciences
+  - global_mmlu_full_id_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_humanities.yaml
new file mode 100644
index 000000000..b9283f55f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_id_humanities
+task:
+  - global_mmlu_full_id_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_other.yaml b/lmms_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_other.yaml
new file mode 100644
index 000000000..74de0f360
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_id_other
+task:
+  - global_mmlu_full_id_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_social_sciences.yaml
new file mode 100644
index 000000000..b8656b6be
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_id_social_sciences
+task:
+  - global_mmlu_full_id_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_stem.yaml b/lmms_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_stem.yaml
new file mode 100644
index 000000000..d0e472764
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_id_stem
+task:
+  - global_mmlu_full_id_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/id/_id_template_yaml b/lmms_eval/tasks/global_mmlu/full/id/_id_template_yaml
new file mode 100644
index 000000000..32d9dc92a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/_id_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: id
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_abstract_algebra.yaml
new file mode 100644
index 000000000..b18c1cd7f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_id_stem_tasks
+task: global_mmlu_full_id_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_anatomy.yaml
new file mode 100644
index 000000000..65b83d9d7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_id_stem_tasks
+task: global_mmlu_full_id_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_astronomy.yaml
new file mode 100644
index 000000000..11f1047c1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_id_stem_tasks
+task: global_mmlu_full_id_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_business_ethics.yaml
new file mode 100644
index 000000000..9ed992f22
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_id_other_tasks
+task: global_mmlu_full_id_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_clinical_knowledge.yaml
new file mode 100644
index 000000000..8baa424fa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_id_other_tasks
+task: global_mmlu_full_id_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_biology.yaml
new file mode 100644
index 000000000..67b9c935b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_id_stem_tasks
+task: global_mmlu_full_id_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_chemistry.yaml
new file mode 100644
index 000000000..3eb5d2280
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_id_stem_tasks
+task: global_mmlu_full_id_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_computer_science.yaml
new file mode 100644
index 000000000..1462945be
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_id_stem_tasks
+task: global_mmlu_full_id_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_mathematics.yaml
new file mode 100644
index 000000000..980627920
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_id_stem_tasks
+task: global_mmlu_full_id_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_medicine.yaml
new file mode 100644
index 000000000..1a2736e89
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_id_other_tasks
+task: global_mmlu_full_id_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_physics.yaml
new file mode 100644
index 000000000..bb88c3f8c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_id_stem_tasks
+task: global_mmlu_full_id_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_computer_security.yaml
new file mode 100644
index 000000000..9764ac3ef
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_id_stem_tasks
+task: global_mmlu_full_id_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_conceptual_physics.yaml
new file mode 100644
index 000000000..c70c111ca
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_id_stem_tasks
+task: global_mmlu_full_id_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_econometrics.yaml
new file mode 100644
index 000000000..7f82a74b2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_id_social_sciences_tasks
+task: global_mmlu_full_id_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_electrical_engineering.yaml
new file mode 100644
index 000000000..3cc2dfba0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_id_stem_tasks
+task: global_mmlu_full_id_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_elementary_mathematics.yaml
new file mode 100644
index 000000000..1d511b4ba
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_id_stem_tasks
+task: global_mmlu_full_id_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_formal_logic.yaml
new file mode 100644
index 000000000..7c6cef13a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_id_humanities_tasks
+task: global_mmlu_full_id_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_global_facts.yaml
new file mode 100644
index 000000000..5e7a44daa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_id_other_tasks
+task: global_mmlu_full_id_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_biology.yaml
new file mode 100644
index 000000000..d39c31abb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_id_stem_tasks
+task: global_mmlu_full_id_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_chemistry.yaml
new file mode 100644
index 000000000..d92d827af
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_id_stem_tasks
+task: global_mmlu_full_id_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_computer_science.yaml
new file mode 100644
index 000000000..ff714ac84
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_id_stem_tasks
+task: global_mmlu_full_id_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_european_history.yaml
new file mode 100644
index 000000000..0d5c81418
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_id_humanities_tasks
+task: global_mmlu_full_id_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_geography.yaml
new file mode 100644
index 000000000..1ad392b3e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_id_social_sciences_tasks
+task: global_mmlu_full_id_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..850d6d828
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_id_social_sciences_tasks
+task: global_mmlu_full_id_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..c1fda5c65
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_id_social_sciences_tasks
+task: global_mmlu_full_id_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_mathematics.yaml
new file mode 100644
index 000000000..8a628ed9a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_id_stem_tasks
+task: global_mmlu_full_id_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_microeconomics.yaml
new file mode 100644
index 000000000..f2c447077
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_id_social_sciences_tasks
+task: global_mmlu_full_id_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_physics.yaml
new file mode 100644
index 000000000..75888a3d0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_id_stem_tasks
+task: global_mmlu_full_id_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_psychology.yaml
new file mode 100644
index 000000000..8a6ff54ba
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_id_social_sciences_tasks
+task: global_mmlu_full_id_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_statistics.yaml
new file mode 100644
index 000000000..ab2058026
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_id_stem_tasks
+task: global_mmlu_full_id_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_us_history.yaml
new file mode 100644
index 000000000..8dee8c319
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_id_humanities_tasks
+task: global_mmlu_full_id_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_world_history.yaml
new file mode 100644
index 000000000..5474c8bac
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_id_humanities_tasks
+task: global_mmlu_full_id_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_human_aging.yaml
new file mode 100644
index 000000000..464ac67f7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_id_other_tasks
+task: global_mmlu_full_id_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_human_sexuality.yaml
new file mode 100644
index 000000000..518cb30c4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_id_social_sciences_tasks
+task: global_mmlu_full_id_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_international_law.yaml
new file mode 100644
index 000000000..90262ada0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_id_humanities_tasks
+task: global_mmlu_full_id_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_jurisprudence.yaml
new file mode 100644
index 000000000..8727ab491
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_id_humanities_tasks
+task: global_mmlu_full_id_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_logical_fallacies.yaml
new file mode 100644
index 000000000..da2c8e6c2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_id_humanities_tasks
+task: global_mmlu_full_id_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_machine_learning.yaml
new file mode 100644
index 000000000..84a30d9d8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_id_stem_tasks
+task: global_mmlu_full_id_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_management.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_management.yaml
new file mode 100644
index 000000000..fdd340bd8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_id_other_tasks
+task: global_mmlu_full_id_management
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_marketing.yaml
new file mode 100644
index 000000000..caf3eb0f2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_id_other_tasks
+task: global_mmlu_full_id_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_medical_genetics.yaml
new file mode 100644
index 000000000..0d649fd3e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_id_other_tasks
+task: global_mmlu_full_id_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_miscellaneous.yaml
new file mode 100644
index 000000000..0811f1b06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_id_other_tasks
+task: global_mmlu_full_id_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_moral_disputes.yaml
new file mode 100644
index 000000000..0a124ded5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_id_humanities_tasks
+task: global_mmlu_full_id_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_moral_scenarios.yaml
new file mode 100644
index 000000000..65dfaea72
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_id_humanities_tasks
+task: global_mmlu_full_id_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_nutrition.yaml
new file mode 100644
index 000000000..804ffc602
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_id_other_tasks
+task: global_mmlu_full_id_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_philosophy.yaml
new file mode 100644
index 000000000..88b37de87
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_id_humanities_tasks
+task: global_mmlu_full_id_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_prehistory.yaml
new file mode 100644
index 000000000..1e851c492
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_id_humanities_tasks
+task: global_mmlu_full_id_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_accounting.yaml
new file mode 100644
index 000000000..d45c95174
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_id_other_tasks
+task: global_mmlu_full_id_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_law.yaml
new file mode 100644
index 000000000..965cbad60
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_id_humanities_tasks
+task: global_mmlu_full_id_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_medicine.yaml
new file mode 100644
index 000000000..fdd02d534
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_id_other_tasks
+task: global_mmlu_full_id_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_psychology.yaml
new file mode 100644
index 000000000..b8d294b27
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_id_social_sciences_tasks
+task: global_mmlu_full_id_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_public_relations.yaml
new file mode 100644
index 000000000..8f772b0cb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_id_social_sciences_tasks
+task: global_mmlu_full_id_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_security_studies.yaml
new file mode 100644
index 000000000..1a73d36bf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_id_social_sciences_tasks
+task: global_mmlu_full_id_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_sociology.yaml
new file mode 100644
index 000000000..715e5c31e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_id_social_sciences_tasks
+task: global_mmlu_full_id_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_us_foreign_policy.yaml
new file mode 100644
index 000000000..59e147aea
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_id_social_sciences_tasks
+task: global_mmlu_full_id_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_virology.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_virology.yaml
new file mode 100644
index 000000000..50225ab5f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_id_other_tasks
+task: global_mmlu_full_id_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_world_religions.yaml
new file mode 100644
index 000000000..0193d12d0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _id_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_id_humanities_tasks
+task: global_mmlu_full_id_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/id/utils.py b/lmms_eval/tasks/global_mmlu/full/id/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/id/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig.yaml b/lmms_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig.yaml
new file mode 100644
index 000000000..d5346e1cb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_ig
+task:
+  - global_mmlu_full_ig_stem
+  - global_mmlu_full_ig_other
+  - global_mmlu_full_ig_social_sciences
+  - global_mmlu_full_ig_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_humanities.yaml
new file mode 100644
index 000000000..6c6ffb612
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ig_humanities
+task:
+  - global_mmlu_full_ig_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_other.yaml b/lmms_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_other.yaml
new file mode 100644
index 000000000..214efed22
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ig_other
+task:
+  - global_mmlu_full_ig_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_social_sciences.yaml
new file mode 100644
index 000000000..e27fe1fa1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ig_social_sciences
+task:
+  - global_mmlu_full_ig_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_stem.yaml b/lmms_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_stem.yaml
new file mode 100644
index 000000000..5dd33b620
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ig_stem
+task:
+  - global_mmlu_full_ig_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/_ig_template_yaml b/lmms_eval/tasks/global_mmlu/full/ig/_ig_template_yaml
new file mode 100644
index 000000000..0832c633b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/_ig_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: ig
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_abstract_algebra.yaml
new file mode 100644
index 000000000..1dbf6c834
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_ig_stem_tasks
+task: global_mmlu_full_ig_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_anatomy.yaml
new file mode 100644
index 000000000..8dc198c25
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_ig_stem_tasks
+task: global_mmlu_full_ig_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_astronomy.yaml
new file mode 100644
index 000000000..078069eb2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_ig_stem_tasks
+task: global_mmlu_full_ig_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_business_ethics.yaml
new file mode 100644
index 000000000..f075e7404
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_ig_other_tasks
+task: global_mmlu_full_ig_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_clinical_knowledge.yaml
new file mode 100644
index 000000000..d41779ade
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_ig_other_tasks
+task: global_mmlu_full_ig_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_biology.yaml
new file mode 100644
index 000000000..5f0e57052
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_ig_stem_tasks
+task: global_mmlu_full_ig_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_chemistry.yaml
new file mode 100644
index 000000000..78e25dc8d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_ig_stem_tasks
+task: global_mmlu_full_ig_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_computer_science.yaml
new file mode 100644
index 000000000..d9894a454
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_ig_stem_tasks
+task: global_mmlu_full_ig_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_mathematics.yaml
new file mode 100644
index 000000000..8976041fd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_ig_stem_tasks
+task: global_mmlu_full_ig_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_medicine.yaml
new file mode 100644
index 000000000..5edaf0d56
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_ig_other_tasks
+task: global_mmlu_full_ig_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_physics.yaml
new file mode 100644
index 000000000..e55c01cbd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_ig_stem_tasks
+task: global_mmlu_full_ig_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_computer_security.yaml
new file mode 100644
index 000000000..5ee7564cb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_ig_stem_tasks
+task: global_mmlu_full_ig_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_conceptual_physics.yaml
new file mode 100644
index 000000000..555d4fa83
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_ig_stem_tasks
+task: global_mmlu_full_ig_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_econometrics.yaml
new file mode 100644
index 000000000..783804b88
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_ig_social_sciences_tasks
+task: global_mmlu_full_ig_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_electrical_engineering.yaml
new file mode 100644
index 000000000..789f95d20
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_ig_stem_tasks
+task: global_mmlu_full_ig_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_elementary_mathematics.yaml
new file mode 100644
index 000000000..7a5c9d2c5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_ig_stem_tasks
+task: global_mmlu_full_ig_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_formal_logic.yaml
new file mode 100644
index 000000000..8f9e426ca
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_ig_humanities_tasks
+task: global_mmlu_full_ig_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_global_facts.yaml
new file mode 100644
index 000000000..d9b7955c6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_ig_other_tasks
+task: global_mmlu_full_ig_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_biology.yaml
new file mode 100644
index 000000000..368bc71dd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_ig_stem_tasks
+task: global_mmlu_full_ig_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_chemistry.yaml
new file mode 100644
index 000000000..1ce77e108
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_ig_stem_tasks
+task: global_mmlu_full_ig_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_computer_science.yaml
new file mode 100644
index 000000000..d859f390f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_ig_stem_tasks
+task: global_mmlu_full_ig_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_european_history.yaml
new file mode 100644
index 000000000..29a93f460
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_ig_humanities_tasks
+task: global_mmlu_full_ig_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_geography.yaml
new file mode 100644
index 000000000..74194a44a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_ig_social_sciences_tasks
+task: global_mmlu_full_ig_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..cd53504d0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_ig_social_sciences_tasks
+task: global_mmlu_full_ig_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..30244a64c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_ig_social_sciences_tasks
+task: global_mmlu_full_ig_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_mathematics.yaml
new file mode 100644
index 000000000..737c0a56c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_ig_stem_tasks
+task: global_mmlu_full_ig_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_microeconomics.yaml
new file mode 100644
index 000000000..c5a2220cc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_ig_social_sciences_tasks
+task: global_mmlu_full_ig_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_physics.yaml
new file mode 100644
index 000000000..a7d4c537f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_ig_stem_tasks
+task: global_mmlu_full_ig_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_psychology.yaml
new file mode 100644
index 000000000..d3051f013
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_ig_social_sciences_tasks
+task: global_mmlu_full_ig_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_statistics.yaml
new file mode 100644
index 000000000..d48410323
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_ig_stem_tasks
+task: global_mmlu_full_ig_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_us_history.yaml
new file mode 100644
index 000000000..61e124fcf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_ig_humanities_tasks
+task: global_mmlu_full_ig_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_world_history.yaml
new file mode 100644
index 000000000..3d83a63d9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_ig_humanities_tasks
+task: global_mmlu_full_ig_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_human_aging.yaml
new file mode 100644
index 000000000..787e3151f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_ig_other_tasks
+task: global_mmlu_full_ig_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_human_sexuality.yaml
new file mode 100644
index 000000000..5c6184591
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_ig_social_sciences_tasks
+task: global_mmlu_full_ig_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_international_law.yaml
new file mode 100644
index 000000000..3a8511d21
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_ig_humanities_tasks
+task: global_mmlu_full_ig_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_jurisprudence.yaml
new file mode 100644
index 000000000..46254ea14
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_ig_humanities_tasks
+task: global_mmlu_full_ig_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_logical_fallacies.yaml
new file mode 100644
index 000000000..2bce75029
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_ig_humanities_tasks
+task: global_mmlu_full_ig_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_machine_learning.yaml
new file mode 100644
index 000000000..93c87fbe6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_ig_stem_tasks
+task: global_mmlu_full_ig_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_management.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_management.yaml
new file mode 100644
index 000000000..780e1c894
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_ig_other_tasks
+task: global_mmlu_full_ig_management
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_marketing.yaml
new file mode 100644
index 000000000..2d30ece9a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_ig_other_tasks
+task: global_mmlu_full_ig_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_medical_genetics.yaml
new file mode 100644
index 000000000..cac197c78
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_ig_other_tasks
+task: global_mmlu_full_ig_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_miscellaneous.yaml
new file mode 100644
index 000000000..a38245103
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_ig_other_tasks
+task: global_mmlu_full_ig_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_moral_disputes.yaml
new file mode 100644
index 000000000..cc545d84c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_ig_humanities_tasks
+task: global_mmlu_full_ig_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_moral_scenarios.yaml
new file mode 100644
index 000000000..60ad22feb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_ig_humanities_tasks
+task: global_mmlu_full_ig_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_nutrition.yaml
new file mode 100644
index 000000000..3cc556070
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_ig_other_tasks
+task: global_mmlu_full_ig_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_philosophy.yaml
new file mode 100644
index 000000000..3f6556324
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_ig_humanities_tasks
+task: global_mmlu_full_ig_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_prehistory.yaml
new file mode 100644
index 000000000..db4affcff
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_ig_humanities_tasks
+task: global_mmlu_full_ig_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_accounting.yaml
new file mode 100644
index 000000000..18d357734
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_ig_other_tasks
+task: global_mmlu_full_ig_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_law.yaml
new file mode 100644
index 000000000..e9db41d00
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_ig_humanities_tasks
+task: global_mmlu_full_ig_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_medicine.yaml
new file mode 100644
index 000000000..7fa28b603
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_ig_other_tasks
+task: global_mmlu_full_ig_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_psychology.yaml
new file mode 100644
index 000000000..639be3813
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_ig_social_sciences_tasks
+task: global_mmlu_full_ig_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_public_relations.yaml
new file mode 100644
index 000000000..d31af09f4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_ig_social_sciences_tasks
+task: global_mmlu_full_ig_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_security_studies.yaml
new file mode 100644
index 000000000..200db46b0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_ig_social_sciences_tasks
+task: global_mmlu_full_ig_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_sociology.yaml
new file mode 100644
index 000000000..65a3e4e1a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_ig_social_sciences_tasks
+task: global_mmlu_full_ig_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_us_foreign_policy.yaml
new file mode 100644
index 000000000..ff0b0505f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_ig_social_sciences_tasks
+task: global_mmlu_full_ig_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_virology.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_virology.yaml
new file mode 100644
index 000000000..b437c82f3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_ig_other_tasks
+task: global_mmlu_full_ig_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_world_religions.yaml
new file mode 100644
index 000000000..6fbc7cfd8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ig_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_ig_humanities_tasks
+task: global_mmlu_full_ig_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/ig/utils.py b/lmms_eval/tasks/global_mmlu/full/ig/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ig/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it.yaml b/lmms_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it.yaml
new file mode 100644
index 000000000..f3bcd6941
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_it
+task:
+  - global_mmlu_full_it_stem
+  - global_mmlu_full_it_other
+  - global_mmlu_full_it_social_sciences
+  - global_mmlu_full_it_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_humanities.yaml
new file mode 100644
index 000000000..3d072ccc5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_it_humanities
+task:
+  - global_mmlu_full_it_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_other.yaml b/lmms_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_other.yaml
new file mode 100644
index 000000000..99fe18cda
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_it_other
+task:
+  - global_mmlu_full_it_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_social_sciences.yaml
new file mode 100644
index 000000000..15a457a55
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_it_social_sciences
+task:
+  - global_mmlu_full_it_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_stem.yaml b/lmms_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_stem.yaml
new file mode 100644
index 000000000..cf7a555dd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_it_stem
+task:
+  - global_mmlu_full_it_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/it/_it_template_yaml b/lmms_eval/tasks/global_mmlu/full/it/_it_template_yaml
new file mode 100644
index 000000000..4798e10a9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/_it_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: it
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_abstract_algebra.yaml
new file mode 100644
index 000000000..f7351c1ee
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_it_stem_tasks
+task: global_mmlu_full_it_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_anatomy.yaml
new file mode 100644
index 000000000..436cd3f66
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_it_stem_tasks
+task: global_mmlu_full_it_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_astronomy.yaml
new file mode 100644
index 000000000..f98f0f20e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_it_stem_tasks
+task: global_mmlu_full_it_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_business_ethics.yaml
new file mode 100644
index 000000000..d9d931fed
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_it_other_tasks
+task: global_mmlu_full_it_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_clinical_knowledge.yaml
new file mode 100644
index 000000000..fe4290240
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_it_other_tasks
+task: global_mmlu_full_it_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_biology.yaml
new file mode 100644
index 000000000..71b8f45e2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_it_stem_tasks
+task: global_mmlu_full_it_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_chemistry.yaml
new file mode 100644
index 000000000..d29bd7585
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_it_stem_tasks
+task: global_mmlu_full_it_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_computer_science.yaml
new file mode 100644
index 000000000..f740d259d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_it_stem_tasks
+task: global_mmlu_full_it_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_mathematics.yaml
new file mode 100644
index 000000000..7568fb7e6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_it_stem_tasks
+task: global_mmlu_full_it_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_medicine.yaml
new file mode 100644
index 000000000..9bfc5ac1b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_it_other_tasks
+task: global_mmlu_full_it_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_physics.yaml
new file mode 100644
index 000000000..2101847ec
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_it_stem_tasks
+task: global_mmlu_full_it_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_computer_security.yaml
new file mode 100644
index 000000000..70b31f9a8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_it_stem_tasks
+task: global_mmlu_full_it_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_conceptual_physics.yaml
new file mode 100644
index 000000000..d8917d409
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_it_stem_tasks
+task: global_mmlu_full_it_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_econometrics.yaml
new file mode 100644
index 000000000..a49352fb7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_it_social_sciences_tasks
+task: global_mmlu_full_it_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_electrical_engineering.yaml
new file mode 100644
index 000000000..27f0c6c33
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_it_stem_tasks
+task: global_mmlu_full_it_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_elementary_mathematics.yaml
new file mode 100644
index 000000000..fd78a52e8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_it_stem_tasks
+task: global_mmlu_full_it_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_formal_logic.yaml
new file mode 100644
index 000000000..8171fcf18
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_it_humanities_tasks
+task: global_mmlu_full_it_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_global_facts.yaml
new file mode 100644
index 000000000..a952ed444
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_it_other_tasks
+task: global_mmlu_full_it_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_biology.yaml
new file mode 100644
index 000000000..939ba7525
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_it_stem_tasks
+task: global_mmlu_full_it_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_chemistry.yaml
new file mode 100644
index 000000000..4524d4dc3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_it_stem_tasks
+task: global_mmlu_full_it_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_computer_science.yaml
new file mode 100644
index 000000000..2dfb1649f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_it_stem_tasks
+task: global_mmlu_full_it_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_european_history.yaml
new file mode 100644
index 000000000..556aaf20a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_it_humanities_tasks
+task: global_mmlu_full_it_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_geography.yaml
new file mode 100644
index 000000000..3c1d5b600
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_it_social_sciences_tasks
+task: global_mmlu_full_it_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..a35b6bac8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_it_social_sciences_tasks
+task: global_mmlu_full_it_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..74c01ccd4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_it_social_sciences_tasks
+task: global_mmlu_full_it_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_mathematics.yaml
new file mode 100644
index 000000000..6bec02c4c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_it_stem_tasks
+task: global_mmlu_full_it_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_microeconomics.yaml
new file mode 100644
index 000000000..551a0f8da
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_it_social_sciences_tasks
+task: global_mmlu_full_it_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_physics.yaml
new file mode 100644
index 000000000..3cf7144bd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_it_stem_tasks
+task: global_mmlu_full_it_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_psychology.yaml
new file mode 100644
index 000000000..17088e513
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_it_social_sciences_tasks
+task: global_mmlu_full_it_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_statistics.yaml
new file mode 100644
index 000000000..f3f35f991
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_it_stem_tasks
+task: global_mmlu_full_it_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_us_history.yaml
new file mode 100644
index 000000000..af222877b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_it_humanities_tasks
+task: global_mmlu_full_it_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_world_history.yaml
new file mode 100644
index 000000000..698ddb5fc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_it_humanities_tasks
+task: global_mmlu_full_it_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_human_aging.yaml
new file mode 100644
index 000000000..6ff49730e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_it_other_tasks
+task: global_mmlu_full_it_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_human_sexuality.yaml
new file mode 100644
index 000000000..58d32fa4a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_it_social_sciences_tasks
+task: global_mmlu_full_it_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_international_law.yaml
new file mode 100644
index 000000000..d7c47e55d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_it_humanities_tasks
+task: global_mmlu_full_it_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_jurisprudence.yaml
new file mode 100644
index 000000000..e100c0e92
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_it_humanities_tasks
+task: global_mmlu_full_it_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_logical_fallacies.yaml
new file mode 100644
index 000000000..a07444a85
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_it_humanities_tasks
+task: global_mmlu_full_it_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_machine_learning.yaml
new file mode 100644
index 000000000..bfd3b7a5a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_it_stem_tasks
+task: global_mmlu_full_it_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_management.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_management.yaml
new file mode 100644
index 000000000..5b5feeacf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_it_other_tasks
+task: global_mmlu_full_it_management
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_marketing.yaml
new file mode 100644
index 000000000..d50b46f44
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_it_other_tasks
+task: global_mmlu_full_it_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_medical_genetics.yaml
new file mode 100644
index 000000000..1b02316ca
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_it_other_tasks
+task: global_mmlu_full_it_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_miscellaneous.yaml
new file mode 100644
index 000000000..b638b50de
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_it_other_tasks
+task: global_mmlu_full_it_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_moral_disputes.yaml
new file mode 100644
index 000000000..520a8beaf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_it_humanities_tasks
+task: global_mmlu_full_it_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_moral_scenarios.yaml
new file mode 100644
index 000000000..abfc73950
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_it_humanities_tasks
+task: global_mmlu_full_it_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_nutrition.yaml
new file mode 100644
index 000000000..cac741527
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_it_other_tasks
+task: global_mmlu_full_it_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_philosophy.yaml
new file mode 100644
index 000000000..a1d94976d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_it_humanities_tasks
+task: global_mmlu_full_it_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_prehistory.yaml
new file mode 100644
index 000000000..74bdec821
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_it_humanities_tasks
+task: global_mmlu_full_it_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_accounting.yaml
new file mode 100644
index 000000000..acf999a29
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_it_other_tasks
+task: global_mmlu_full_it_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_law.yaml
new file mode 100644
index 000000000..1ec4f58e3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_it_humanities_tasks
+task: global_mmlu_full_it_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_medicine.yaml
new file mode 100644
index 000000000..8b53cdacb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_it_other_tasks
+task: global_mmlu_full_it_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_psychology.yaml
new file mode 100644
index 000000000..9b7a24f39
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_it_social_sciences_tasks
+task: global_mmlu_full_it_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_public_relations.yaml
new file mode 100644
index 000000000..727cf4b6e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_it_social_sciences_tasks
+task: global_mmlu_full_it_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_security_studies.yaml
new file mode 100644
index 000000000..90fd186c5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_it_social_sciences_tasks
+task: global_mmlu_full_it_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_sociology.yaml
new file mode 100644
index 000000000..dbc77935a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_it_social_sciences_tasks
+task: global_mmlu_full_it_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_us_foreign_policy.yaml
new file mode 100644
index 000000000..e2a923b6b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_it_social_sciences_tasks
+task: global_mmlu_full_it_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_virology.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_virology.yaml
new file mode 100644
index 000000000..72758a56c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_it_other_tasks
+task: global_mmlu_full_it_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_world_religions.yaml
new file mode 100644
index 000000000..e4491c51b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _it_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_it_humanities_tasks
+task: global_mmlu_full_it_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/it/utils.py b/lmms_eval/tasks/global_mmlu/full/it/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/it/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja.yaml b/lmms_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja.yaml
new file mode 100644
index 000000000..af25573df
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_ja
+task:
+  - global_mmlu_full_ja_stem
+  - global_mmlu_full_ja_other
+  - global_mmlu_full_ja_social_sciences
+  - global_mmlu_full_ja_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_humanities.yaml
new file mode 100644
index 000000000..a063eb0b6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ja_humanities
+task:
+  - global_mmlu_full_ja_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_other.yaml b/lmms_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_other.yaml
new file mode 100644
index 000000000..1f9b95ed6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ja_other
+task:
+  - global_mmlu_full_ja_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_social_sciences.yaml
new file mode 100644
index 000000000..4207fea43
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ja_social_sciences
+task:
+  - global_mmlu_full_ja_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_stem.yaml b/lmms_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_stem.yaml
new file mode 100644
index 000000000..7ca6ed1c0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ja_stem
+task:
+  - global_mmlu_full_ja_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/_ja_template_yaml b/lmms_eval/tasks/global_mmlu/full/ja/_ja_template_yaml
new file mode 100644
index 000000000..591725e38
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/_ja_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: ja
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_abstract_algebra.yaml
new file mode 100644
index 000000000..4b65a75be
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_ja_stem_tasks
+task: global_mmlu_full_ja_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_anatomy.yaml
new file mode 100644
index 000000000..e735aa349
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_ja_stem_tasks
+task: global_mmlu_full_ja_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_astronomy.yaml
new file mode 100644
index 000000000..a0ba89475
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_ja_stem_tasks
+task: global_mmlu_full_ja_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_business_ethics.yaml
new file mode 100644
index 000000000..c39d286c7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_ja_other_tasks
+task: global_mmlu_full_ja_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_clinical_knowledge.yaml
new file mode 100644
index 000000000..27d09b888
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_ja_other_tasks
+task: global_mmlu_full_ja_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_biology.yaml
new file mode 100644
index 000000000..15e26a51a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_ja_stem_tasks
+task: global_mmlu_full_ja_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_chemistry.yaml
new file mode 100644
index 000000000..52c92423e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_ja_stem_tasks
+task: global_mmlu_full_ja_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_computer_science.yaml
new file mode 100644
index 000000000..a91a7d617
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_ja_stem_tasks
+task: global_mmlu_full_ja_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_mathematics.yaml
new file mode 100644
index 000000000..67dcd2a02
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_ja_stem_tasks
+task: global_mmlu_full_ja_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_medicine.yaml
new file mode 100644
index 000000000..c55ab2a25
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_ja_other_tasks
+task: global_mmlu_full_ja_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_physics.yaml
new file mode 100644
index 000000000..5413c86d5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_ja_stem_tasks
+task: global_mmlu_full_ja_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_computer_security.yaml
new file mode 100644
index 000000000..276f214e7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_ja_stem_tasks
+task: global_mmlu_full_ja_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_conceptual_physics.yaml
new file mode 100644
index 000000000..f823ac44b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_ja_stem_tasks
+task: global_mmlu_full_ja_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_econometrics.yaml
new file mode 100644
index 000000000..dbc6846a7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_ja_social_sciences_tasks
+task: global_mmlu_full_ja_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_electrical_engineering.yaml
new file mode 100644
index 000000000..ba7295759
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_ja_stem_tasks
+task: global_mmlu_full_ja_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_elementary_mathematics.yaml
new file mode 100644
index 000000000..138071048
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_ja_stem_tasks
+task: global_mmlu_full_ja_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_formal_logic.yaml
new file mode 100644
index 000000000..d88d5685a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_ja_humanities_tasks
+task: global_mmlu_full_ja_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_global_facts.yaml
new file mode 100644
index 000000000..64cb2b9ee
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_ja_other_tasks
+task: global_mmlu_full_ja_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_biology.yaml
new file mode 100644
index 000000000..b88adf905
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_ja_stem_tasks
+task: global_mmlu_full_ja_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_chemistry.yaml
new file mode 100644
index 000000000..eef67cc77
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_ja_stem_tasks
+task: global_mmlu_full_ja_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_computer_science.yaml
new file mode 100644
index 000000000..c90e5fb76
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_ja_stem_tasks
+task: global_mmlu_full_ja_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_european_history.yaml
new file mode 100644
index 000000000..8318099ac
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_ja_humanities_tasks
+task: global_mmlu_full_ja_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_geography.yaml
new file mode 100644
index 000000000..4ed5a6209
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_ja_social_sciences_tasks
+task: global_mmlu_full_ja_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..6ec0ab842
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_ja_social_sciences_tasks
+task: global_mmlu_full_ja_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..4bdd9555d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_ja_social_sciences_tasks
+task: global_mmlu_full_ja_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_mathematics.yaml
new file mode 100644
index 000000000..3abfd81b1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_ja_stem_tasks
+task: global_mmlu_full_ja_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_microeconomics.yaml
new file mode 100644
index 000000000..483161a60
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_ja_social_sciences_tasks
+task: global_mmlu_full_ja_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_physics.yaml
new file mode 100644
index 000000000..702092af4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_ja_stem_tasks
+task: global_mmlu_full_ja_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_psychology.yaml
new file mode 100644
index 000000000..1b7ce92ee
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_ja_social_sciences_tasks
+task: global_mmlu_full_ja_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_statistics.yaml
new file mode 100644
index 000000000..c68acb8e8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_ja_stem_tasks
+task: global_mmlu_full_ja_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_us_history.yaml
new file mode 100644
index 000000000..b1b918332
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_ja_humanities_tasks
+task: global_mmlu_full_ja_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_world_history.yaml
new file mode 100644
index 000000000..2dee1f89e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_ja_humanities_tasks
+task: global_mmlu_full_ja_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_human_aging.yaml
new file mode 100644
index 000000000..3612a7eec
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_ja_other_tasks
+task: global_mmlu_full_ja_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_human_sexuality.yaml
new file mode 100644
index 000000000..b70204fbc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_ja_social_sciences_tasks
+task: global_mmlu_full_ja_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_international_law.yaml
new file mode 100644
index 000000000..77ed3c97d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_ja_humanities_tasks
+task: global_mmlu_full_ja_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_jurisprudence.yaml
new file mode 100644
index 000000000..f8fbb261f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_ja_humanities_tasks
+task: global_mmlu_full_ja_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_logical_fallacies.yaml
new file mode 100644
index 000000000..58d4afccb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_ja_humanities_tasks
+task: global_mmlu_full_ja_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_machine_learning.yaml
new file mode 100644
index 000000000..e664390f0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_ja_stem_tasks
+task: global_mmlu_full_ja_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_management.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_management.yaml
new file mode 100644
index 000000000..cf495ae22
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_ja_other_tasks
+task: global_mmlu_full_ja_management
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_marketing.yaml
new file mode 100644
index 000000000..1349771ef
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_ja_other_tasks
+task: global_mmlu_full_ja_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_medical_genetics.yaml
new file mode 100644
index 000000000..1b513ac4f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_ja_other_tasks
+task: global_mmlu_full_ja_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_miscellaneous.yaml
new file mode 100644
index 000000000..81659bf77
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_ja_other_tasks
+task: global_mmlu_full_ja_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_moral_disputes.yaml
new file mode 100644
index 000000000..2e77694bd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_ja_humanities_tasks
+task: global_mmlu_full_ja_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_moral_scenarios.yaml
new file mode 100644
index 000000000..f322376da
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_ja_humanities_tasks
+task: global_mmlu_full_ja_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_nutrition.yaml
new file mode 100644
index 000000000..1d58fb0e6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_ja_other_tasks
+task: global_mmlu_full_ja_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_philosophy.yaml
new file mode 100644
index 000000000..238653619
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_ja_humanities_tasks
+task: global_mmlu_full_ja_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_prehistory.yaml
new file mode 100644
index 000000000..a044bf99c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_ja_humanities_tasks
+task: global_mmlu_full_ja_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_accounting.yaml
new file mode 100644
index 000000000..b828e0e28
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_ja_other_tasks
+task: global_mmlu_full_ja_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_law.yaml
new file mode 100644
index 000000000..7aafb6c4c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_ja_humanities_tasks
+task: global_mmlu_full_ja_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_medicine.yaml
new file mode 100644
index 000000000..b0cf9905e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_ja_other_tasks
+task: global_mmlu_full_ja_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_psychology.yaml
new file mode 100644
index 000000000..e5ef36c18
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_ja_social_sciences_tasks
+task: global_mmlu_full_ja_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_public_relations.yaml
new file mode 100644
index 000000000..565439e6d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_ja_social_sciences_tasks
+task: global_mmlu_full_ja_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_security_studies.yaml
new file mode 100644
index 000000000..f7d21bd64
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_ja_social_sciences_tasks
+task: global_mmlu_full_ja_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_sociology.yaml
new file mode 100644
index 000000000..5cc44c1c1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_ja_social_sciences_tasks
+task: global_mmlu_full_ja_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_us_foreign_policy.yaml
new file mode 100644
index 000000000..8ebdb14a8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_ja_social_sciences_tasks
+task: global_mmlu_full_ja_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_virology.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_virology.yaml
new file mode 100644
index 000000000..d6f833672
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_ja_other_tasks
+task: global_mmlu_full_ja_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_world_religions.yaml
new file mode 100644
index 000000000..23e66e063
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_ja_humanities_tasks
+task: global_mmlu_full_ja_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/ja/utils.py b/lmms_eval/tasks/global_mmlu/full/ja/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ja/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko.yaml b/lmms_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko.yaml
new file mode 100644
index 000000000..8b9b41970
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_ko
+task:
+  - global_mmlu_full_ko_stem
+  - global_mmlu_full_ko_other
+  - global_mmlu_full_ko_social_sciences
+  - global_mmlu_full_ko_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_humanities.yaml
new file mode 100644
index 000000000..c76906434
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ko_humanities
+task:
+  - global_mmlu_full_ko_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_other.yaml b/lmms_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_other.yaml
new file mode 100644
index 000000000..8990ae957
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ko_other
+task:
+  - global_mmlu_full_ko_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_social_sciences.yaml
new file mode 100644
index 000000000..0bbfad7f2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ko_social_sciences
+task:
+  - global_mmlu_full_ko_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_stem.yaml b/lmms_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_stem.yaml
new file mode 100644
index 000000000..18b7f17ba
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ko_stem
+task:
+  - global_mmlu_full_ko_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/_ko_template_yaml b/lmms_eval/tasks/global_mmlu/full/ko/_ko_template_yaml
new file mode 100644
index 000000000..11700a261
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/_ko_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: ko
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_abstract_algebra.yaml
new file mode 100644
index 000000000..5959d7881
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_ko_stem_tasks
+task: global_mmlu_full_ko_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_anatomy.yaml
new file mode 100644
index 000000000..ebb90860f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_ko_stem_tasks
+task: global_mmlu_full_ko_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_astronomy.yaml
new file mode 100644
index 000000000..670846b32
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_ko_stem_tasks
+task: global_mmlu_full_ko_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_business_ethics.yaml
new file mode 100644
index 000000000..1a44e4302
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_ko_other_tasks
+task: global_mmlu_full_ko_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_clinical_knowledge.yaml
new file mode 100644
index 000000000..e9e29697c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_ko_other_tasks
+task: global_mmlu_full_ko_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_biology.yaml
new file mode 100644
index 000000000..fc3644680
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_ko_stem_tasks
+task: global_mmlu_full_ko_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_chemistry.yaml
new file mode 100644
index 000000000..2eb0f416f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_ko_stem_tasks
+task: global_mmlu_full_ko_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_computer_science.yaml
new file mode 100644
index 000000000..044f1eff5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_ko_stem_tasks
+task: global_mmlu_full_ko_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_mathematics.yaml
new file mode 100644
index 000000000..9929097ce
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_ko_stem_tasks
+task: global_mmlu_full_ko_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_medicine.yaml
new file mode 100644
index 000000000..b78c24e6f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_ko_other_tasks
+task: global_mmlu_full_ko_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_physics.yaml
new file mode 100644
index 000000000..20c3fb20a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_ko_stem_tasks
+task: global_mmlu_full_ko_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_computer_security.yaml
new file mode 100644
index 000000000..1f9545724
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_ko_stem_tasks
+task: global_mmlu_full_ko_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_conceptual_physics.yaml
new file mode 100644
index 000000000..f79989750
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_ko_stem_tasks
+task: global_mmlu_full_ko_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_econometrics.yaml
new file mode 100644
index 000000000..79c35ed7f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_ko_social_sciences_tasks
+task: global_mmlu_full_ko_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_electrical_engineering.yaml
new file mode 100644
index 000000000..1444a2495
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_ko_stem_tasks
+task: global_mmlu_full_ko_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_elementary_mathematics.yaml
new file mode 100644
index 000000000..8bec91b99
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_ko_stem_tasks
+task: global_mmlu_full_ko_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_formal_logic.yaml
new file mode 100644
index 000000000..1cf310923
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_ko_humanities_tasks
+task: global_mmlu_full_ko_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_global_facts.yaml
new file mode 100644
index 000000000..2a5f7bd54
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_ko_other_tasks
+task: global_mmlu_full_ko_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_biology.yaml
new file mode 100644
index 000000000..bdaed574d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_ko_stem_tasks
+task: global_mmlu_full_ko_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_chemistry.yaml
new file mode 100644
index 000000000..193a064c9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_ko_stem_tasks
+task: global_mmlu_full_ko_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_computer_science.yaml
new file mode 100644
index 000000000..2d2ad648a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_ko_stem_tasks
+task: global_mmlu_full_ko_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_european_history.yaml
new file mode 100644
index 000000000..a48b602dd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_ko_humanities_tasks
+task: global_mmlu_full_ko_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_geography.yaml
new file mode 100644
index 000000000..cc9c20eb6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_ko_social_sciences_tasks
+task: global_mmlu_full_ko_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..e86a27fa5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_ko_social_sciences_tasks
+task: global_mmlu_full_ko_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..4b947f7f3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_ko_social_sciences_tasks
+task: global_mmlu_full_ko_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_mathematics.yaml
new file mode 100644
index 000000000..9184ad9ce
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_ko_stem_tasks
+task: global_mmlu_full_ko_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_microeconomics.yaml
new file mode 100644
index 000000000..50b6a1503
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_ko_social_sciences_tasks
+task: global_mmlu_full_ko_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_physics.yaml
new file mode 100644
index 000000000..974e3b03d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_ko_stem_tasks
+task: global_mmlu_full_ko_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_psychology.yaml
new file mode 100644
index 000000000..e617e8cdd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_ko_social_sciences_tasks
+task: global_mmlu_full_ko_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_statistics.yaml
new file mode 100644
index 000000000..1a0105961
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_ko_stem_tasks
+task: global_mmlu_full_ko_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_us_history.yaml
new file mode 100644
index 000000000..a696675df
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_ko_humanities_tasks
+task: global_mmlu_full_ko_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_world_history.yaml
new file mode 100644
index 000000000..eca86cbe4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_ko_humanities_tasks
+task: global_mmlu_full_ko_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_human_aging.yaml
new file mode 100644
index 000000000..69e3a2df4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_ko_other_tasks
+task: global_mmlu_full_ko_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_human_sexuality.yaml
new file mode 100644
index 000000000..ed3e99fc5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_ko_social_sciences_tasks
+task: global_mmlu_full_ko_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_international_law.yaml
new file mode 100644
index 000000000..651f389c2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_ko_humanities_tasks
+task: global_mmlu_full_ko_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_jurisprudence.yaml
new file mode 100644
index 000000000..001807ebc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_ko_humanities_tasks
+task: global_mmlu_full_ko_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_logical_fallacies.yaml
new file mode 100644
index 000000000..01eec477c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_ko_humanities_tasks
+task: global_mmlu_full_ko_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_machine_learning.yaml
new file mode 100644
index 000000000..c1126c6b7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_ko_stem_tasks
+task: global_mmlu_full_ko_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_management.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_management.yaml
new file mode 100644
index 000000000..3b8332705
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_ko_other_tasks
+task: global_mmlu_full_ko_management
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_marketing.yaml
new file mode 100644
index 000000000..3cce25c09
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_ko_other_tasks
+task: global_mmlu_full_ko_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_medical_genetics.yaml
new file mode 100644
index 000000000..65df17861
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_ko_other_tasks
+task: global_mmlu_full_ko_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_miscellaneous.yaml
new file mode 100644
index 000000000..04b71e2a7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_ko_other_tasks
+task: global_mmlu_full_ko_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_moral_disputes.yaml
new file mode 100644
index 000000000..3f1e7fa1b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_ko_humanities_tasks
+task: global_mmlu_full_ko_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_moral_scenarios.yaml
new file mode 100644
index 000000000..c657543aa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_ko_humanities_tasks
+task: global_mmlu_full_ko_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_nutrition.yaml
new file mode 100644
index 000000000..dff6450f9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_ko_other_tasks
+task: global_mmlu_full_ko_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_philosophy.yaml
new file mode 100644
index 000000000..21f058af0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_ko_humanities_tasks
+task: global_mmlu_full_ko_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_prehistory.yaml
new file mode 100644
index 000000000..56aedae91
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_ko_humanities_tasks
+task: global_mmlu_full_ko_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_accounting.yaml
new file mode 100644
index 000000000..24f83b232
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_ko_other_tasks
+task: global_mmlu_full_ko_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_law.yaml
new file mode 100644
index 000000000..ece9dc5c6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_ko_humanities_tasks
+task: global_mmlu_full_ko_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_medicine.yaml
new file mode 100644
index 000000000..439309570
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_ko_other_tasks
+task: global_mmlu_full_ko_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_psychology.yaml
new file mode 100644
index 000000000..98ff6520d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_ko_social_sciences_tasks
+task: global_mmlu_full_ko_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_public_relations.yaml
new file mode 100644
index 000000000..1a5b07f7e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_ko_social_sciences_tasks
+task: global_mmlu_full_ko_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_security_studies.yaml
new file mode 100644
index 000000000..3663391a8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_ko_social_sciences_tasks
+task: global_mmlu_full_ko_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_sociology.yaml
new file mode 100644
index 000000000..902b4443e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_ko_social_sciences_tasks
+task: global_mmlu_full_ko_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_us_foreign_policy.yaml
new file mode 100644
index 000000000..36e1794ce
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_ko_social_sciences_tasks
+task: global_mmlu_full_ko_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_virology.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_virology.yaml
new file mode 100644
index 000000000..64b58d6a9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_ko_other_tasks
+task: global_mmlu_full_ko_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_world_religions.yaml
new file mode 100644
index 000000000..7289671f5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_ko_humanities_tasks
+task: global_mmlu_full_ko_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/ko/utils.py b/lmms_eval/tasks/global_mmlu/full/ko/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ko/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky.yaml b/lmms_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky.yaml
new file mode 100644
index 000000000..33b2a4df9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_ky
+task:
+  - global_mmlu_full_ky_stem
+  - global_mmlu_full_ky_other
+  - global_mmlu_full_ky_social_sciences
+  - global_mmlu_full_ky_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_humanities.yaml
new file mode 100644
index 000000000..1e0368c2b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ky_humanities
+task:
+  - global_mmlu_full_ky_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_other.yaml b/lmms_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_other.yaml
new file mode 100644
index 000000000..1bfc89ab6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ky_other
+task:
+  - global_mmlu_full_ky_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_social_sciences.yaml
new file mode 100644
index 000000000..3ae756c4b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ky_social_sciences
+task:
+  - global_mmlu_full_ky_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_stem.yaml b/lmms_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_stem.yaml
new file mode 100644
index 000000000..817456fcb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ky_stem
+task:
+  - global_mmlu_full_ky_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/_ky_template_yaml b/lmms_eval/tasks/global_mmlu/full/ky/_ky_template_yaml
new file mode 100644
index 000000000..63f888238
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/_ky_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: ky
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_abstract_algebra.yaml
new file mode 100644
index 000000000..21338a569
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_ky_stem_tasks
+task: global_mmlu_full_ky_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_anatomy.yaml
new file mode 100644
index 000000000..df2635483
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_ky_stem_tasks
+task: global_mmlu_full_ky_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_astronomy.yaml
new file mode 100644
index 000000000..5e0f6abab
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_ky_stem_tasks
+task: global_mmlu_full_ky_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_business_ethics.yaml
new file mode 100644
index 000000000..17656dc63
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_ky_other_tasks
+task: global_mmlu_full_ky_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_clinical_knowledge.yaml
new file mode 100644
index 000000000..8c053b884
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_ky_other_tasks
+task: global_mmlu_full_ky_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_biology.yaml
new file mode 100644
index 000000000..364921063
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_ky_stem_tasks
+task: global_mmlu_full_ky_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_chemistry.yaml
new file mode 100644
index 000000000..cb9f85868
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_ky_stem_tasks
+task: global_mmlu_full_ky_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_computer_science.yaml
new file mode 100644
index 000000000..e4b15b540
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_ky_stem_tasks
+task: global_mmlu_full_ky_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_mathematics.yaml
new file mode 100644
index 000000000..f5657b66b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_ky_stem_tasks
+task: global_mmlu_full_ky_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_medicine.yaml
new file mode 100644
index 000000000..f1e0c25e6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_ky_other_tasks
+task: global_mmlu_full_ky_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_physics.yaml
new file mode 100644
index 000000000..fac1d80f3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_ky_stem_tasks
+task: global_mmlu_full_ky_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_computer_security.yaml
new file mode 100644
index 000000000..e35718d02
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_ky_stem_tasks
+task: global_mmlu_full_ky_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_conceptual_physics.yaml
new file mode 100644
index 000000000..f165ec619
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_ky_stem_tasks
+task: global_mmlu_full_ky_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_econometrics.yaml
new file mode 100644
index 000000000..48670c7f8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_ky_social_sciences_tasks
+task: global_mmlu_full_ky_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_electrical_engineering.yaml
new file mode 100644
index 000000000..29d241428
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_ky_stem_tasks
+task: global_mmlu_full_ky_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_elementary_mathematics.yaml
new file mode 100644
index 000000000..9aa7f81b4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_ky_stem_tasks
+task: global_mmlu_full_ky_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_formal_logic.yaml
new file mode 100644
index 000000000..70a5bd860
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_ky_humanities_tasks
+task: global_mmlu_full_ky_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_global_facts.yaml
new file mode 100644
index 000000000..f678c0d13
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_ky_other_tasks
+task: global_mmlu_full_ky_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_biology.yaml
new file mode 100644
index 000000000..750bc68b8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_ky_stem_tasks
+task: global_mmlu_full_ky_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_chemistry.yaml
new file mode 100644
index 000000000..7700e37fa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_ky_stem_tasks
+task: global_mmlu_full_ky_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_computer_science.yaml
new file mode 100644
index 000000000..c805fc4e9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_ky_stem_tasks
+task: global_mmlu_full_ky_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_european_history.yaml
new file mode 100644
index 000000000..01c67f8e3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_ky_humanities_tasks
+task: global_mmlu_full_ky_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_geography.yaml
new file mode 100644
index 000000000..ccc5c8b73
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_ky_social_sciences_tasks
+task: global_mmlu_full_ky_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..02ea66ef6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_ky_social_sciences_tasks
+task: global_mmlu_full_ky_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..f693296db
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_ky_social_sciences_tasks
+task: global_mmlu_full_ky_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_mathematics.yaml
new file mode 100644
index 000000000..b05e27992
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_ky_stem_tasks
+task: global_mmlu_full_ky_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_microeconomics.yaml
new file mode 100644
index 000000000..d596290fa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_ky_social_sciences_tasks
+task: global_mmlu_full_ky_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_physics.yaml
new file mode 100644
index 000000000..3f71865c2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_ky_stem_tasks
+task: global_mmlu_full_ky_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_psychology.yaml
new file mode 100644
index 000000000..635873a1f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_ky_social_sciences_tasks
+task: global_mmlu_full_ky_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_statistics.yaml
new file mode 100644
index 000000000..df8cfefbc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_ky_stem_tasks
+task: global_mmlu_full_ky_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_us_history.yaml
new file mode 100644
index 000000000..3c75f5341
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_ky_humanities_tasks
+task: global_mmlu_full_ky_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_world_history.yaml
new file mode 100644
index 000000000..e73edcbea
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_ky_humanities_tasks
+task: global_mmlu_full_ky_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_human_aging.yaml
new file mode 100644
index 000000000..f4e662a56
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_ky_other_tasks
+task: global_mmlu_full_ky_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_human_sexuality.yaml
new file mode 100644
index 000000000..8c2556da1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_ky_social_sciences_tasks
+task: global_mmlu_full_ky_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_international_law.yaml
new file mode 100644
index 000000000..2af161909
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_ky_humanities_tasks
+task: global_mmlu_full_ky_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_jurisprudence.yaml
new file mode 100644
index 000000000..f0994cc2d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_ky_humanities_tasks
+task: global_mmlu_full_ky_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_logical_fallacies.yaml
new file mode 100644
index 000000000..f7933a77b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_ky_humanities_tasks
+task: global_mmlu_full_ky_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_machine_learning.yaml
new file mode 100644
index 000000000..f6e525a18
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_ky_stem_tasks
+task: global_mmlu_full_ky_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_management.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_management.yaml
new file mode 100644
index 000000000..03f70aa02
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_ky_other_tasks
+task: global_mmlu_full_ky_management
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_marketing.yaml
new file mode 100644
index 000000000..72ced7988
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_ky_other_tasks
+task: global_mmlu_full_ky_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_medical_genetics.yaml
new file mode 100644
index 000000000..371e4b217
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_ky_other_tasks
+task: global_mmlu_full_ky_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_miscellaneous.yaml
new file mode 100644
index 000000000..e693ab8b9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_ky_other_tasks
+task: global_mmlu_full_ky_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_moral_disputes.yaml
new file mode 100644
index 000000000..ccafcb1e0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_ky_humanities_tasks
+task: global_mmlu_full_ky_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_moral_scenarios.yaml
new file mode 100644
index 000000000..16c19b29e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_ky_humanities_tasks
+task: global_mmlu_full_ky_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_nutrition.yaml
new file mode 100644
index 000000000..f6c00cb0e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_ky_other_tasks
+task: global_mmlu_full_ky_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_philosophy.yaml
new file mode 100644
index 000000000..6d6d242b0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_ky_humanities_tasks
+task: global_mmlu_full_ky_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_prehistory.yaml
new file mode 100644
index 000000000..4ff2e08d4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_ky_humanities_tasks
+task: global_mmlu_full_ky_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_accounting.yaml
new file mode 100644
index 000000000..37c6a8922
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_ky_other_tasks
+task: global_mmlu_full_ky_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_law.yaml
new file mode 100644
index 000000000..9b4fea0e5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_ky_humanities_tasks
+task: global_mmlu_full_ky_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_medicine.yaml
new file mode 100644
index 000000000..8a6ef0b53
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_ky_other_tasks
+task: global_mmlu_full_ky_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_psychology.yaml
new file mode 100644
index 000000000..dce1b6d2d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_ky_social_sciences_tasks
+task: global_mmlu_full_ky_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_public_relations.yaml
new file mode 100644
index 000000000..168cae744
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_ky_social_sciences_tasks
+task: global_mmlu_full_ky_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_security_studies.yaml
new file mode 100644
index 000000000..1e24b8167
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_ky_social_sciences_tasks
+task: global_mmlu_full_ky_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_sociology.yaml
new file mode 100644
index 000000000..7d1ad9593
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_ky_social_sciences_tasks
+task: global_mmlu_full_ky_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_us_foreign_policy.yaml
new file mode 100644
index 000000000..36cd7e206
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_ky_social_sciences_tasks
+task: global_mmlu_full_ky_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_virology.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_virology.yaml
new file mode 100644
index 000000000..e2a779151
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_ky_other_tasks
+task: global_mmlu_full_ky_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_world_religions.yaml
new file mode 100644
index 000000000..563c13976
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ky_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_ky_humanities_tasks
+task: global_mmlu_full_ky_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/ky/utils.py b/lmms_eval/tasks/global_mmlu/full/ky/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ky/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt.yaml b/lmms_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt.yaml
new file mode 100644
index 000000000..dd2cd37d3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_lt
+task:
+  - global_mmlu_full_lt_stem
+  - global_mmlu_full_lt_other
+  - global_mmlu_full_lt_social_sciences
+  - global_mmlu_full_lt_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_humanities.yaml
new file mode 100644
index 000000000..48ad351f7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_lt_humanities
+task:
+  - global_mmlu_full_lt_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_other.yaml b/lmms_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_other.yaml
new file mode 100644
index 000000000..8f63c35ad
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_lt_other
+task:
+  - global_mmlu_full_lt_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_social_sciences.yaml
new file mode 100644
index 000000000..9ababd6d1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_lt_social_sciences
+task:
+  - global_mmlu_full_lt_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_stem.yaml b/lmms_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_stem.yaml
new file mode 100644
index 000000000..1a59e6833
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_lt_stem
+task:
+  - global_mmlu_full_lt_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/_lt_template_yaml b/lmms_eval/tasks/global_mmlu/full/lt/_lt_template_yaml
new file mode 100644
index 000000000..8b9253380
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/_lt_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: lt
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_abstract_algebra.yaml
new file mode 100644
index 000000000..76b96844f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_lt_stem_tasks
+task: global_mmlu_full_lt_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_anatomy.yaml
new file mode 100644
index 000000000..527c71074
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_lt_stem_tasks
+task: global_mmlu_full_lt_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_astronomy.yaml
new file mode 100644
index 000000000..419b89e30
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_lt_stem_tasks
+task: global_mmlu_full_lt_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_business_ethics.yaml
new file mode 100644
index 000000000..c51daa226
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_lt_other_tasks
+task: global_mmlu_full_lt_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_clinical_knowledge.yaml
new file mode 100644
index 000000000..e02327740
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_lt_other_tasks
+task: global_mmlu_full_lt_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_biology.yaml
new file mode 100644
index 000000000..c6fea6f85
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_lt_stem_tasks
+task: global_mmlu_full_lt_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_chemistry.yaml
new file mode 100644
index 000000000..93b9a5610
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_lt_stem_tasks
+task: global_mmlu_full_lt_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_computer_science.yaml
new file mode 100644
index 000000000..8d0dcfdd9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_lt_stem_tasks
+task: global_mmlu_full_lt_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_mathematics.yaml
new file mode 100644
index 000000000..8d33b747d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_lt_stem_tasks
+task: global_mmlu_full_lt_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_medicine.yaml
new file mode 100644
index 000000000..ad74dbb15
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_lt_other_tasks
+task: global_mmlu_full_lt_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_physics.yaml
new file mode 100644
index 000000000..3c69754b3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_lt_stem_tasks
+task: global_mmlu_full_lt_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_computer_security.yaml
new file mode 100644
index 000000000..d78f3a545
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_lt_stem_tasks
+task: global_mmlu_full_lt_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_conceptual_physics.yaml
new file mode 100644
index 000000000..3e7b5e49e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_lt_stem_tasks
+task: global_mmlu_full_lt_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_econometrics.yaml
new file mode 100644
index 000000000..6d0085ab9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_lt_social_sciences_tasks
+task: global_mmlu_full_lt_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_electrical_engineering.yaml
new file mode 100644
index 000000000..284dfe9e2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_lt_stem_tasks
+task: global_mmlu_full_lt_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_elementary_mathematics.yaml
new file mode 100644
index 000000000..7e9a01030
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_lt_stem_tasks
+task: global_mmlu_full_lt_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_formal_logic.yaml
new file mode 100644
index 000000000..ec9a665b5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_lt_humanities_tasks
+task: global_mmlu_full_lt_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_global_facts.yaml
new file mode 100644
index 000000000..d81a94704
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_lt_other_tasks
+task: global_mmlu_full_lt_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_biology.yaml
new file mode 100644
index 000000000..139376cc7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_lt_stem_tasks
+task: global_mmlu_full_lt_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_chemistry.yaml
new file mode 100644
index 000000000..87112d8c7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_lt_stem_tasks
+task: global_mmlu_full_lt_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_computer_science.yaml
new file mode 100644
index 000000000..2324bb288
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_lt_stem_tasks
+task: global_mmlu_full_lt_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_european_history.yaml
new file mode 100644
index 000000000..5f365fab1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_lt_humanities_tasks
+task: global_mmlu_full_lt_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_geography.yaml
new file mode 100644
index 000000000..e3a6f9214
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_lt_social_sciences_tasks
+task: global_mmlu_full_lt_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..526b68ed9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_lt_social_sciences_tasks
+task: global_mmlu_full_lt_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..e14b1dcea
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_lt_social_sciences_tasks
+task: global_mmlu_full_lt_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_mathematics.yaml
new file mode 100644
index 000000000..1cdf5c90b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_lt_stem_tasks
+task: global_mmlu_full_lt_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_microeconomics.yaml
new file mode 100644
index 000000000..a2f2a2103
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_lt_social_sciences_tasks
+task: global_mmlu_full_lt_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_physics.yaml
new file mode 100644
index 000000000..bd3637091
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_lt_stem_tasks
+task: global_mmlu_full_lt_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_psychology.yaml
new file mode 100644
index 000000000..aad65a136
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_lt_social_sciences_tasks
+task: global_mmlu_full_lt_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_statistics.yaml
new file mode 100644
index 000000000..6dd6d699a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_lt_stem_tasks
+task: global_mmlu_full_lt_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_us_history.yaml
new file mode 100644
index 000000000..5fb0ee1e5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_lt_humanities_tasks
+task: global_mmlu_full_lt_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_world_history.yaml
new file mode 100644
index 000000000..75f2769ad
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_lt_humanities_tasks
+task: global_mmlu_full_lt_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_human_aging.yaml
new file mode 100644
index 000000000..beb27e9bd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_lt_other_tasks
+task: global_mmlu_full_lt_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_human_sexuality.yaml
new file mode 100644
index 000000000..c9d952c39
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_lt_social_sciences_tasks
+task: global_mmlu_full_lt_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_international_law.yaml
new file mode 100644
index 000000000..f77adf9b3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_lt_humanities_tasks
+task: global_mmlu_full_lt_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_jurisprudence.yaml
new file mode 100644
index 000000000..e6be84fde
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_lt_humanities_tasks
+task: global_mmlu_full_lt_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_logical_fallacies.yaml
new file mode 100644
index 000000000..ad597b27a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_lt_humanities_tasks
+task: global_mmlu_full_lt_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_machine_learning.yaml
new file mode 100644
index 000000000..eb06a871b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_lt_stem_tasks
+task: global_mmlu_full_lt_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_management.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_management.yaml
new file mode 100644
index 000000000..e1885ad3f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_lt_other_tasks
+task: global_mmlu_full_lt_management
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_marketing.yaml
new file mode 100644
index 000000000..2dc830890
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_lt_other_tasks
+task: global_mmlu_full_lt_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_medical_genetics.yaml
new file mode 100644
index 000000000..b67d321e4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_lt_other_tasks
+task: global_mmlu_full_lt_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_miscellaneous.yaml
new file mode 100644
index 000000000..2c744613c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_lt_other_tasks
+task: global_mmlu_full_lt_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_moral_disputes.yaml
new file mode 100644
index 000000000..09e6f044f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_lt_humanities_tasks
+task: global_mmlu_full_lt_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_moral_scenarios.yaml
new file mode 100644
index 000000000..bb8dd3307
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_lt_humanities_tasks
+task: global_mmlu_full_lt_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_nutrition.yaml
new file mode 100644
index 000000000..0b1a85566
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_lt_other_tasks
+task: global_mmlu_full_lt_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_philosophy.yaml
new file mode 100644
index 000000000..aab1d556c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_lt_humanities_tasks
+task: global_mmlu_full_lt_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_prehistory.yaml
new file mode 100644
index 000000000..ac93dd6cc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_lt_humanities_tasks
+task: global_mmlu_full_lt_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_accounting.yaml
new file mode 100644
index 000000000..6be78ec39
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_lt_other_tasks
+task: global_mmlu_full_lt_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_law.yaml
new file mode 100644
index 000000000..60b6cdcc0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_lt_humanities_tasks
+task: global_mmlu_full_lt_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_medicine.yaml
new file mode 100644
index 000000000..dd899676e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_lt_other_tasks
+task: global_mmlu_full_lt_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_psychology.yaml
new file mode 100644
index 000000000..bd796e8b5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_lt_social_sciences_tasks
+task: global_mmlu_full_lt_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_public_relations.yaml
new file mode 100644
index 000000000..3c6e5f397
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_lt_social_sciences_tasks
+task: global_mmlu_full_lt_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_security_studies.yaml
new file mode 100644
index 000000000..9eb9957d2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_lt_social_sciences_tasks
+task: global_mmlu_full_lt_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_sociology.yaml
new file mode 100644
index 000000000..2e17f95a6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_lt_social_sciences_tasks
+task: global_mmlu_full_lt_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_us_foreign_policy.yaml
new file mode 100644
index 000000000..d39bb63ca
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_lt_social_sciences_tasks
+task: global_mmlu_full_lt_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_virology.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_virology.yaml
new file mode 100644
index 000000000..b8482a618
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_lt_other_tasks
+task: global_mmlu_full_lt_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_world_religions.yaml
new file mode 100644
index 000000000..a86af60dc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _lt_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_lt_humanities_tasks
+task: global_mmlu_full_lt_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/lt/utils.py b/lmms_eval/tasks/global_mmlu/full/lt/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/lt/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg.yaml b/lmms_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg.yaml
new file mode 100644
index 000000000..bcc6cd81a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_mg
+task:
+  - global_mmlu_full_mg_stem
+  - global_mmlu_full_mg_other
+  - global_mmlu_full_mg_social_sciences
+  - global_mmlu_full_mg_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_humanities.yaml
new file mode 100644
index 000000000..76b08f6c7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_mg_humanities
+task:
+  - global_mmlu_full_mg_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_other.yaml b/lmms_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_other.yaml
new file mode 100644
index 000000000..0006af4c9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_mg_other
+task:
+  - global_mmlu_full_mg_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_social_sciences.yaml
new file mode 100644
index 000000000..9cfe4f5b0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_mg_social_sciences
+task:
+  - global_mmlu_full_mg_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_stem.yaml b/lmms_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_stem.yaml
new file mode 100644
index 000000000..bdc719d10
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_mg_stem
+task:
+  - global_mmlu_full_mg_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/_mg_template_yaml b/lmms_eval/tasks/global_mmlu/full/mg/_mg_template_yaml
new file mode 100644
index 000000000..4aa97b27f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/_mg_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: mg
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_abstract_algebra.yaml
new file mode 100644
index 000000000..bea850ed1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_mg_stem_tasks
+task: global_mmlu_full_mg_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_anatomy.yaml
new file mode 100644
index 000000000..1cf6c1169
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_mg_stem_tasks
+task: global_mmlu_full_mg_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_astronomy.yaml
new file mode 100644
index 000000000..df582b272
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_mg_stem_tasks
+task: global_mmlu_full_mg_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_business_ethics.yaml
new file mode 100644
index 000000000..a63513426
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_mg_other_tasks
+task: global_mmlu_full_mg_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_clinical_knowledge.yaml
new file mode 100644
index 000000000..21003af50
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_mg_other_tasks
+task: global_mmlu_full_mg_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_biology.yaml
new file mode 100644
index 000000000..d305ca94e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_mg_stem_tasks
+task: global_mmlu_full_mg_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_chemistry.yaml
new file mode 100644
index 000000000..7ccaffb9d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_mg_stem_tasks
+task: global_mmlu_full_mg_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_computer_science.yaml
new file mode 100644
index 000000000..248f72c3d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_mg_stem_tasks
+task: global_mmlu_full_mg_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_mathematics.yaml
new file mode 100644
index 000000000..fb817aaeb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_mg_stem_tasks
+task: global_mmlu_full_mg_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_medicine.yaml
new file mode 100644
index 000000000..4fdbee03b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_mg_other_tasks
+task: global_mmlu_full_mg_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_physics.yaml
new file mode 100644
index 000000000..493bdf87b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_mg_stem_tasks
+task: global_mmlu_full_mg_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_computer_security.yaml
new file mode 100644
index 000000000..c5054eb63
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_mg_stem_tasks
+task: global_mmlu_full_mg_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_conceptual_physics.yaml
new file mode 100644
index 000000000..44a13a70e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_mg_stem_tasks
+task: global_mmlu_full_mg_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_econometrics.yaml
new file mode 100644
index 000000000..2c5d029ad
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_mg_social_sciences_tasks
+task: global_mmlu_full_mg_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_electrical_engineering.yaml
new file mode 100644
index 000000000..1e5ece337
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_mg_stem_tasks
+task: global_mmlu_full_mg_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_elementary_mathematics.yaml
new file mode 100644
index 000000000..4d62c7585
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_mg_stem_tasks
+task: global_mmlu_full_mg_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_formal_logic.yaml
new file mode 100644
index 000000000..e5dc67d05
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_mg_humanities_tasks
+task: global_mmlu_full_mg_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_global_facts.yaml
new file mode 100644
index 000000000..2712e9b46
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_mg_other_tasks
+task: global_mmlu_full_mg_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_biology.yaml
new file mode 100644
index 000000000..c58957e03
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_mg_stem_tasks
+task: global_mmlu_full_mg_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_chemistry.yaml
new file mode 100644
index 000000000..707b73564
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_mg_stem_tasks
+task: global_mmlu_full_mg_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_computer_science.yaml
new file mode 100644
index 000000000..d7afd5a74
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_mg_stem_tasks
+task: global_mmlu_full_mg_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_european_history.yaml
new file mode 100644
index 000000000..b6391ee44
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_mg_humanities_tasks
+task: global_mmlu_full_mg_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_geography.yaml
new file mode 100644
index 000000000..eb7014a6b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_mg_social_sciences_tasks
+task: global_mmlu_full_mg_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..74c5fc18d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_mg_social_sciences_tasks
+task: global_mmlu_full_mg_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..24631ff31
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_mg_social_sciences_tasks
+task: global_mmlu_full_mg_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_mathematics.yaml
new file mode 100644
index 000000000..b9db4a0c2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_mg_stem_tasks
+task: global_mmlu_full_mg_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_microeconomics.yaml
new file mode 100644
index 000000000..f321b06ac
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_mg_social_sciences_tasks
+task: global_mmlu_full_mg_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_physics.yaml
new file mode 100644
index 000000000..bc25971e4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_mg_stem_tasks
+task: global_mmlu_full_mg_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_psychology.yaml
new file mode 100644
index 000000000..42cc39a8e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_mg_social_sciences_tasks
+task: global_mmlu_full_mg_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_statistics.yaml
new file mode 100644
index 000000000..08cf86712
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_mg_stem_tasks
+task: global_mmlu_full_mg_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_us_history.yaml
new file mode 100644
index 000000000..87314a572
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_mg_humanities_tasks
+task: global_mmlu_full_mg_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_world_history.yaml
new file mode 100644
index 000000000..c341a243e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_mg_humanities_tasks
+task: global_mmlu_full_mg_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_human_aging.yaml
new file mode 100644
index 000000000..15375f9f0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_mg_other_tasks
+task: global_mmlu_full_mg_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_human_sexuality.yaml
new file mode 100644
index 000000000..21419b9b6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_mg_social_sciences_tasks
+task: global_mmlu_full_mg_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_international_law.yaml
new file mode 100644
index 000000000..9d4813393
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_mg_humanities_tasks
+task: global_mmlu_full_mg_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_jurisprudence.yaml
new file mode 100644
index 000000000..f083a0ab5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_mg_humanities_tasks
+task: global_mmlu_full_mg_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_logical_fallacies.yaml
new file mode 100644
index 000000000..57e2e7318
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_mg_humanities_tasks
+task: global_mmlu_full_mg_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_machine_learning.yaml
new file mode 100644
index 000000000..7609a09f6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_mg_stem_tasks
+task: global_mmlu_full_mg_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_management.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_management.yaml
new file mode 100644
index 000000000..becfe4b30
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_mg_other_tasks
+task: global_mmlu_full_mg_management
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_marketing.yaml
new file mode 100644
index 000000000..3765002b8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_mg_other_tasks
+task: global_mmlu_full_mg_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_medical_genetics.yaml
new file mode 100644
index 000000000..3f023ccd4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_mg_other_tasks
+task: global_mmlu_full_mg_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_miscellaneous.yaml
new file mode 100644
index 000000000..2993999d5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_mg_other_tasks
+task: global_mmlu_full_mg_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_moral_disputes.yaml
new file mode 100644
index 000000000..fd430a0a1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_mg_humanities_tasks
+task: global_mmlu_full_mg_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_moral_scenarios.yaml
new file mode 100644
index 000000000..c1b16e865
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_mg_humanities_tasks
+task: global_mmlu_full_mg_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_nutrition.yaml
new file mode 100644
index 000000000..ab471f42c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_mg_other_tasks
+task: global_mmlu_full_mg_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_philosophy.yaml
new file mode 100644
index 000000000..f598830ed
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_mg_humanities_tasks
+task: global_mmlu_full_mg_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_prehistory.yaml
new file mode 100644
index 000000000..330f1f52a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_mg_humanities_tasks
+task: global_mmlu_full_mg_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_accounting.yaml
new file mode 100644
index 000000000..694118d11
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_mg_other_tasks
+task: global_mmlu_full_mg_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_law.yaml
new file mode 100644
index 000000000..fb6df92ac
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_mg_humanities_tasks
+task: global_mmlu_full_mg_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_medicine.yaml
new file mode 100644
index 000000000..1de72b6b8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_mg_other_tasks
+task: global_mmlu_full_mg_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_psychology.yaml
new file mode 100644
index 000000000..f922e1624
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_mg_social_sciences_tasks
+task: global_mmlu_full_mg_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_public_relations.yaml
new file mode 100644
index 000000000..c829b89dc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_mg_social_sciences_tasks
+task: global_mmlu_full_mg_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_security_studies.yaml
new file mode 100644
index 000000000..362b4dbd1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_mg_social_sciences_tasks
+task: global_mmlu_full_mg_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_sociology.yaml
new file mode 100644
index 000000000..f0638cdb6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_mg_social_sciences_tasks
+task: global_mmlu_full_mg_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_us_foreign_policy.yaml
new file mode 100644
index 000000000..8ead541a4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_mg_social_sciences_tasks
+task: global_mmlu_full_mg_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_virology.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_virology.yaml
new file mode 100644
index 000000000..1ca09027f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_mg_other_tasks
+task: global_mmlu_full_mg_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_world_religions.yaml
new file mode 100644
index 000000000..2bb64d701
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _mg_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_mg_humanities_tasks
+task: global_mmlu_full_mg_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/mg/utils.py b/lmms_eval/tasks/global_mmlu/full/mg/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/mg/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms.yaml b/lmms_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms.yaml
new file mode 100644
index 000000000..a322dad5b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_ms
+task:
+  - global_mmlu_full_ms_stem
+  - global_mmlu_full_ms_other
+  - global_mmlu_full_ms_social_sciences
+  - global_mmlu_full_ms_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_humanities.yaml
new file mode 100644
index 000000000..0641187b8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ms_humanities
+task:
+  - global_mmlu_full_ms_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_other.yaml b/lmms_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_other.yaml
new file mode 100644
index 000000000..3d14420c0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ms_other
+task:
+  - global_mmlu_full_ms_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_social_sciences.yaml
new file mode 100644
index 000000000..3db339d79
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ms_social_sciences
+task:
+  - global_mmlu_full_ms_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_stem.yaml b/lmms_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_stem.yaml
new file mode 100644
index 000000000..68908e16a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ms_stem
+task:
+  - global_mmlu_full_ms_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/_ms_template_yaml b/lmms_eval/tasks/global_mmlu/full/ms/_ms_template_yaml
new file mode 100644
index 000000000..ba7502649
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/_ms_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: ms
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_abstract_algebra.yaml
new file mode 100644
index 000000000..ec791f2a0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_ms_stem_tasks
+task: global_mmlu_full_ms_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_anatomy.yaml
new file mode 100644
index 000000000..35038beaf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_ms_stem_tasks
+task: global_mmlu_full_ms_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_astronomy.yaml
new file mode 100644
index 000000000..79fdcbdd0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_ms_stem_tasks
+task: global_mmlu_full_ms_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_business_ethics.yaml
new file mode 100644
index 000000000..ffd6195a9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_ms_other_tasks
+task: global_mmlu_full_ms_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_clinical_knowledge.yaml
new file mode 100644
index 000000000..4c69b82eb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_ms_other_tasks
+task: global_mmlu_full_ms_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_biology.yaml
new file mode 100644
index 000000000..582194794
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_ms_stem_tasks
+task: global_mmlu_full_ms_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_chemistry.yaml
new file mode 100644
index 000000000..35514b834
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_ms_stem_tasks
+task: global_mmlu_full_ms_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_computer_science.yaml
new file mode 100644
index 000000000..5e242b8bc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_ms_stem_tasks
+task: global_mmlu_full_ms_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_mathematics.yaml
new file mode 100644
index 000000000..07e107995
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_ms_stem_tasks
+task: global_mmlu_full_ms_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_medicine.yaml
new file mode 100644
index 000000000..82822217f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_ms_other_tasks
+task: global_mmlu_full_ms_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_physics.yaml
new file mode 100644
index 000000000..be20fa6cf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_ms_stem_tasks
+task: global_mmlu_full_ms_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_computer_security.yaml
new file mode 100644
index 000000000..2e886b501
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_ms_stem_tasks
+task: global_mmlu_full_ms_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_conceptual_physics.yaml
new file mode 100644
index 000000000..2a2fb6da5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_ms_stem_tasks
+task: global_mmlu_full_ms_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_econometrics.yaml
new file mode 100644
index 000000000..efdffabf9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_ms_social_sciences_tasks
+task: global_mmlu_full_ms_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_electrical_engineering.yaml
new file mode 100644
index 000000000..80eba2e47
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_ms_stem_tasks
+task: global_mmlu_full_ms_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_elementary_mathematics.yaml
new file mode 100644
index 000000000..1e6caf268
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_ms_stem_tasks
+task: global_mmlu_full_ms_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_formal_logic.yaml
new file mode 100644
index 000000000..591476623
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_ms_humanities_tasks
+task: global_mmlu_full_ms_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_global_facts.yaml
new file mode 100644
index 000000000..6ac76cadf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_ms_other_tasks
+task: global_mmlu_full_ms_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_biology.yaml
new file mode 100644
index 000000000..6be8ccfe1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_ms_stem_tasks
+task: global_mmlu_full_ms_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_chemistry.yaml
new file mode 100644
index 000000000..f01c29b67
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_ms_stem_tasks
+task: global_mmlu_full_ms_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_computer_science.yaml
new file mode 100644
index 000000000..b18e8cf85
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_ms_stem_tasks
+task: global_mmlu_full_ms_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_european_history.yaml
new file mode 100644
index 000000000..fdb41802d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_ms_humanities_tasks
+task: global_mmlu_full_ms_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_geography.yaml
new file mode 100644
index 000000000..c4e44a60e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_ms_social_sciences_tasks
+task: global_mmlu_full_ms_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..0ebbfe6f4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_ms_social_sciences_tasks
+task: global_mmlu_full_ms_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..f28f9a5dd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_ms_social_sciences_tasks
+task: global_mmlu_full_ms_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_mathematics.yaml
new file mode 100644
index 000000000..50a2552d0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_ms_stem_tasks
+task: global_mmlu_full_ms_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_microeconomics.yaml
new file mode 100644
index 000000000..6747cd9da
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_ms_social_sciences_tasks
+task: global_mmlu_full_ms_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_physics.yaml
new file mode 100644
index 000000000..aef3fee82
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_ms_stem_tasks
+task: global_mmlu_full_ms_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_psychology.yaml
new file mode 100644
index 000000000..3e8641e97
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_ms_social_sciences_tasks
+task: global_mmlu_full_ms_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_statistics.yaml
new file mode 100644
index 000000000..4aa7ba005
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_ms_stem_tasks
+task: global_mmlu_full_ms_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_us_history.yaml
new file mode 100644
index 000000000..e6d1faabf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_ms_humanities_tasks
+task: global_mmlu_full_ms_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_world_history.yaml
new file mode 100644
index 000000000..4caf7e547
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_ms_humanities_tasks
+task: global_mmlu_full_ms_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_human_aging.yaml
new file mode 100644
index 000000000..5b2b5c5f4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_ms_other_tasks
+task: global_mmlu_full_ms_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_human_sexuality.yaml
new file mode 100644
index 000000000..2ddef17a7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_ms_social_sciences_tasks
+task: global_mmlu_full_ms_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_international_law.yaml
new file mode 100644
index 000000000..61795f589
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_ms_humanities_tasks
+task: global_mmlu_full_ms_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_jurisprudence.yaml
new file mode 100644
index 000000000..f2e96706b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_ms_humanities_tasks
+task: global_mmlu_full_ms_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_logical_fallacies.yaml
new file mode 100644
index 000000000..1d142bde6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_ms_humanities_tasks
+task: global_mmlu_full_ms_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_machine_learning.yaml
new file mode 100644
index 000000000..947240560
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_ms_stem_tasks
+task: global_mmlu_full_ms_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_management.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_management.yaml
new file mode 100644
index 000000000..8ca04a13f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_ms_other_tasks
+task: global_mmlu_full_ms_management
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_marketing.yaml
new file mode 100644
index 000000000..ec0e44626
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_ms_other_tasks
+task: global_mmlu_full_ms_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_medical_genetics.yaml
new file mode 100644
index 000000000..0f2b1eec7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_ms_other_tasks
+task: global_mmlu_full_ms_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_miscellaneous.yaml
new file mode 100644
index 000000000..65da952e2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_ms_other_tasks
+task: global_mmlu_full_ms_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_moral_disputes.yaml
new file mode 100644
index 000000000..399035f2c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_ms_humanities_tasks
+task: global_mmlu_full_ms_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_moral_scenarios.yaml
new file mode 100644
index 000000000..3bc74baa4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_ms_humanities_tasks
+task: global_mmlu_full_ms_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_nutrition.yaml
new file mode 100644
index 000000000..300de6771
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_ms_other_tasks
+task: global_mmlu_full_ms_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_philosophy.yaml
new file mode 100644
index 000000000..8f6eceaee
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_ms_humanities_tasks
+task: global_mmlu_full_ms_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_prehistory.yaml
new file mode 100644
index 000000000..4c624fec5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_ms_humanities_tasks
+task: global_mmlu_full_ms_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_accounting.yaml
new file mode 100644
index 000000000..9a06e7f92
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_ms_other_tasks
+task: global_mmlu_full_ms_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_law.yaml
new file mode 100644
index 000000000..b3d5921a8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_ms_humanities_tasks
+task: global_mmlu_full_ms_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_medicine.yaml
new file mode 100644
index 000000000..0d9a58b7b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_ms_other_tasks
+task: global_mmlu_full_ms_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_psychology.yaml
new file mode 100644
index 000000000..7f51baec9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_ms_social_sciences_tasks
+task: global_mmlu_full_ms_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_public_relations.yaml
new file mode 100644
index 000000000..c07cbdee8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_ms_social_sciences_tasks
+task: global_mmlu_full_ms_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_security_studies.yaml
new file mode 100644
index 000000000..651cb72d4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_ms_social_sciences_tasks
+task: global_mmlu_full_ms_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_sociology.yaml
new file mode 100644
index 000000000..5aeb7efa4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_ms_social_sciences_tasks
+task: global_mmlu_full_ms_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_us_foreign_policy.yaml
new file mode 100644
index 000000000..ecbf57059
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_ms_social_sciences_tasks
+task: global_mmlu_full_ms_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_virology.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_virology.yaml
new file mode 100644
index 000000000..fbdd5e256
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_ms_other_tasks
+task: global_mmlu_full_ms_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_world_religions.yaml
new file mode 100644
index 000000000..32b350294
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ms_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_ms_humanities_tasks
+task: global_mmlu_full_ms_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/ms/utils.py b/lmms_eval/tasks/global_mmlu/full/ms/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ms/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne.yaml b/lmms_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne.yaml
new file mode 100644
index 000000000..7e9c77ce6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_ne
+task:
+  - global_mmlu_full_ne_stem
+  - global_mmlu_full_ne_other
+  - global_mmlu_full_ne_social_sciences
+  - global_mmlu_full_ne_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_humanities.yaml
new file mode 100644
index 000000000..fef749dbf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ne_humanities
+task:
+  - global_mmlu_full_ne_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_other.yaml b/lmms_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_other.yaml
new file mode 100644
index 000000000..0d3dfbd46
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ne_other
+task:
+  - global_mmlu_full_ne_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_social_sciences.yaml
new file mode 100644
index 000000000..f1f09f002
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ne_social_sciences
+task:
+  - global_mmlu_full_ne_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_stem.yaml b/lmms_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_stem.yaml
new file mode 100644
index 000000000..eebc1cac3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ne_stem
+task:
+  - global_mmlu_full_ne_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/_ne_template_yaml b/lmms_eval/tasks/global_mmlu/full/ne/_ne_template_yaml
new file mode 100644
index 000000000..25f8daecd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/_ne_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: ne
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_abstract_algebra.yaml
new file mode 100644
index 000000000..48bf7bb18
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_ne_stem_tasks
+task: global_mmlu_full_ne_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_anatomy.yaml
new file mode 100644
index 000000000..0f66f8ec7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_ne_stem_tasks
+task: global_mmlu_full_ne_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_astronomy.yaml
new file mode 100644
index 000000000..a02aaf30c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_ne_stem_tasks
+task: global_mmlu_full_ne_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_business_ethics.yaml
new file mode 100644
index 000000000..d87f5b985
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_ne_other_tasks
+task: global_mmlu_full_ne_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_clinical_knowledge.yaml
new file mode 100644
index 000000000..f27eb4e2d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_ne_other_tasks
+task: global_mmlu_full_ne_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_biology.yaml
new file mode 100644
index 000000000..d26edef8a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_ne_stem_tasks
+task: global_mmlu_full_ne_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_chemistry.yaml
new file mode 100644
index 000000000..88b8bd86f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_ne_stem_tasks
+task: global_mmlu_full_ne_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_computer_science.yaml
new file mode 100644
index 000000000..51909ffc8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_ne_stem_tasks
+task: global_mmlu_full_ne_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_mathematics.yaml
new file mode 100644
index 000000000..40b9cb796
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_ne_stem_tasks
+task: global_mmlu_full_ne_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_medicine.yaml
new file mode 100644
index 000000000..81f81f844
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_ne_other_tasks
+task: global_mmlu_full_ne_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_physics.yaml
new file mode 100644
index 000000000..09798c096
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_ne_stem_tasks
+task: global_mmlu_full_ne_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_computer_security.yaml
new file mode 100644
index 000000000..49d89dd43
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_ne_stem_tasks
+task: global_mmlu_full_ne_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_conceptual_physics.yaml
new file mode 100644
index 000000000..94bfec4a9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_ne_stem_tasks
+task: global_mmlu_full_ne_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_econometrics.yaml
new file mode 100644
index 000000000..81d6ed98e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_ne_social_sciences_tasks
+task: global_mmlu_full_ne_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_electrical_engineering.yaml
new file mode 100644
index 000000000..73ad1a34a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_ne_stem_tasks
+task: global_mmlu_full_ne_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_elementary_mathematics.yaml
new file mode 100644
index 000000000..cbc3bacd0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_ne_stem_tasks
+task: global_mmlu_full_ne_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_formal_logic.yaml
new file mode 100644
index 000000000..225da2fb5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_ne_humanities_tasks
+task: global_mmlu_full_ne_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_global_facts.yaml
new file mode 100644
index 000000000..6f5e9f1b4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_ne_other_tasks
+task: global_mmlu_full_ne_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_biology.yaml
new file mode 100644
index 000000000..a8c0436a4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_ne_stem_tasks
+task: global_mmlu_full_ne_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_chemistry.yaml
new file mode 100644
index 000000000..405661c7e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_ne_stem_tasks
+task: global_mmlu_full_ne_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_computer_science.yaml
new file mode 100644
index 000000000..6cff5ba64
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_ne_stem_tasks
+task: global_mmlu_full_ne_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_european_history.yaml
new file mode 100644
index 000000000..4f7eb3ee3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_ne_humanities_tasks
+task: global_mmlu_full_ne_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_geography.yaml
new file mode 100644
index 000000000..0453e51d9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_ne_social_sciences_tasks
+task: global_mmlu_full_ne_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..057101007
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_ne_social_sciences_tasks
+task: global_mmlu_full_ne_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..fd68d5f04
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_ne_social_sciences_tasks
+task: global_mmlu_full_ne_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_mathematics.yaml
new file mode 100644
index 000000000..39ef0a58f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_ne_stem_tasks
+task: global_mmlu_full_ne_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_microeconomics.yaml
new file mode 100644
index 000000000..535a39182
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_ne_social_sciences_tasks
+task: global_mmlu_full_ne_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_physics.yaml
new file mode 100644
index 000000000..f355dad25
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_ne_stem_tasks
+task: global_mmlu_full_ne_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_psychology.yaml
new file mode 100644
index 000000000..a52d7a017
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_ne_social_sciences_tasks
+task: global_mmlu_full_ne_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_statistics.yaml
new file mode 100644
index 000000000..5a2564200
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_ne_stem_tasks
+task: global_mmlu_full_ne_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_us_history.yaml
new file mode 100644
index 000000000..9e1199b17
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_ne_humanities_tasks
+task: global_mmlu_full_ne_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_world_history.yaml
new file mode 100644
index 000000000..afc2135b3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_ne_humanities_tasks
+task: global_mmlu_full_ne_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_human_aging.yaml
new file mode 100644
index 000000000..184505346
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_ne_other_tasks
+task: global_mmlu_full_ne_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_human_sexuality.yaml
new file mode 100644
index 000000000..7d23b8397
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_ne_social_sciences_tasks
+task: global_mmlu_full_ne_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_international_law.yaml
new file mode 100644
index 000000000..5be599d20
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_ne_humanities_tasks
+task: global_mmlu_full_ne_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_jurisprudence.yaml
new file mode 100644
index 000000000..180a397c6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_ne_humanities_tasks
+task: global_mmlu_full_ne_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_logical_fallacies.yaml
new file mode 100644
index 000000000..3aa369a96
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_ne_humanities_tasks
+task: global_mmlu_full_ne_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_machine_learning.yaml
new file mode 100644
index 000000000..4e08abdac
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_ne_stem_tasks
+task: global_mmlu_full_ne_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_management.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_management.yaml
new file mode 100644
index 000000000..e44c5be63
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_ne_other_tasks
+task: global_mmlu_full_ne_management
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_marketing.yaml
new file mode 100644
index 000000000..10f7daa27
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_ne_other_tasks
+task: global_mmlu_full_ne_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_medical_genetics.yaml
new file mode 100644
index 000000000..8139b1f7a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_ne_other_tasks
+task: global_mmlu_full_ne_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_miscellaneous.yaml
new file mode 100644
index 000000000..cb1bf9050
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_ne_other_tasks
+task: global_mmlu_full_ne_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_moral_disputes.yaml
new file mode 100644
index 000000000..1b74fb365
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_ne_humanities_tasks
+task: global_mmlu_full_ne_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_moral_scenarios.yaml
new file mode 100644
index 000000000..91f8f06c5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_ne_humanities_tasks
+task: global_mmlu_full_ne_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_nutrition.yaml
new file mode 100644
index 000000000..575f0e455
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_ne_other_tasks
+task: global_mmlu_full_ne_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_philosophy.yaml
new file mode 100644
index 000000000..95fdd0eb3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_ne_humanities_tasks
+task: global_mmlu_full_ne_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_prehistory.yaml
new file mode 100644
index 000000000..e6e5c706c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_ne_humanities_tasks
+task: global_mmlu_full_ne_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_accounting.yaml
new file mode 100644
index 000000000..718cedee0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_ne_other_tasks
+task: global_mmlu_full_ne_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_law.yaml
new file mode 100644
index 000000000..89c70160a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_ne_humanities_tasks
+task: global_mmlu_full_ne_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_medicine.yaml
new file mode 100644
index 000000000..a366e0c43
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_ne_other_tasks
+task: global_mmlu_full_ne_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_psychology.yaml
new file mode 100644
index 000000000..649e53435
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_ne_social_sciences_tasks
+task: global_mmlu_full_ne_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_public_relations.yaml
new file mode 100644
index 000000000..37f2ddeab
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_ne_social_sciences_tasks
+task: global_mmlu_full_ne_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_security_studies.yaml
new file mode 100644
index 000000000..55f809041
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_ne_social_sciences_tasks
+task: global_mmlu_full_ne_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_sociology.yaml
new file mode 100644
index 000000000..78161d5a1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_ne_social_sciences_tasks
+task: global_mmlu_full_ne_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_us_foreign_policy.yaml
new file mode 100644
index 000000000..c38f59c40
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_ne_social_sciences_tasks
+task: global_mmlu_full_ne_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_virology.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_virology.yaml
new file mode 100644
index 000000000..0c15808fd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_ne_other_tasks
+task: global_mmlu_full_ne_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_world_religions.yaml
new file mode 100644
index 000000000..5c6163f12
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ne_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_ne_humanities_tasks
+task: global_mmlu_full_ne_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/ne/utils.py b/lmms_eval/tasks/global_mmlu/full/ne/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ne/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl.yaml b/lmms_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl.yaml
new file mode 100644
index 000000000..c293a558a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_nl
+task:
+  - global_mmlu_full_nl_stem
+  - global_mmlu_full_nl_other
+  - global_mmlu_full_nl_social_sciences
+  - global_mmlu_full_nl_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_humanities.yaml
new file mode 100644
index 000000000..656a421bf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_nl_humanities
+task:
+  - global_mmlu_full_nl_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_other.yaml b/lmms_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_other.yaml
new file mode 100644
index 000000000..23a42201e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_nl_other
+task:
+  - global_mmlu_full_nl_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_social_sciences.yaml
new file mode 100644
index 000000000..afba5678d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_nl_social_sciences
+task:
+  - global_mmlu_full_nl_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_stem.yaml b/lmms_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_stem.yaml
new file mode 100644
index 000000000..9658b13ed
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_nl_stem
+task:
+  - global_mmlu_full_nl_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/_nl_template_yaml b/lmms_eval/tasks/global_mmlu/full/nl/_nl_template_yaml
new file mode 100644
index 000000000..39efbfd17
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/_nl_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: nl
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_abstract_algebra.yaml
new file mode 100644
index 000000000..458a36140
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_nl_stem_tasks
+task: global_mmlu_full_nl_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_anatomy.yaml
new file mode 100644
index 000000000..e4cbd90ef
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_nl_stem_tasks
+task: global_mmlu_full_nl_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_astronomy.yaml
new file mode 100644
index 000000000..84cdf5781
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_nl_stem_tasks
+task: global_mmlu_full_nl_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_business_ethics.yaml
new file mode 100644
index 000000000..f75776f24
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_nl_other_tasks
+task: global_mmlu_full_nl_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_clinical_knowledge.yaml
new file mode 100644
index 000000000..6e963d0f0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_nl_other_tasks
+task: global_mmlu_full_nl_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_biology.yaml
new file mode 100644
index 000000000..e4a3660b0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_nl_stem_tasks
+task: global_mmlu_full_nl_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_chemistry.yaml
new file mode 100644
index 000000000..fa9faed92
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_nl_stem_tasks
+task: global_mmlu_full_nl_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_computer_science.yaml
new file mode 100644
index 000000000..b603c3091
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_nl_stem_tasks
+task: global_mmlu_full_nl_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_mathematics.yaml
new file mode 100644
index 000000000..f55207ea4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_nl_stem_tasks
+task: global_mmlu_full_nl_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_medicine.yaml
new file mode 100644
index 000000000..5cdda1b77
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_nl_other_tasks
+task: global_mmlu_full_nl_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_physics.yaml
new file mode 100644
index 000000000..26d70230f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_nl_stem_tasks
+task: global_mmlu_full_nl_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_computer_security.yaml
new file mode 100644
index 000000000..01a8a7474
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_nl_stem_tasks
+task: global_mmlu_full_nl_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_conceptual_physics.yaml
new file mode 100644
index 000000000..cccd26665
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_nl_stem_tasks
+task: global_mmlu_full_nl_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_econometrics.yaml
new file mode 100644
index 000000000..22ad59bd4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_nl_social_sciences_tasks
+task: global_mmlu_full_nl_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_electrical_engineering.yaml
new file mode 100644
index 000000000..3aca226fd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_nl_stem_tasks
+task: global_mmlu_full_nl_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_elementary_mathematics.yaml
new file mode 100644
index 000000000..2118a1d5d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_nl_stem_tasks
+task: global_mmlu_full_nl_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_formal_logic.yaml
new file mode 100644
index 000000000..5fd86105d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_nl_humanities_tasks
+task: global_mmlu_full_nl_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_global_facts.yaml
new file mode 100644
index 000000000..d7147d517
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_nl_other_tasks
+task: global_mmlu_full_nl_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_biology.yaml
new file mode 100644
index 000000000..271b54f65
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_nl_stem_tasks
+task: global_mmlu_full_nl_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_chemistry.yaml
new file mode 100644
index 000000000..921abd174
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_nl_stem_tasks
+task: global_mmlu_full_nl_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_computer_science.yaml
new file mode 100644
index 000000000..ea190bea8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_nl_stem_tasks
+task: global_mmlu_full_nl_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_european_history.yaml
new file mode 100644
index 000000000..c348d482a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_nl_humanities_tasks
+task: global_mmlu_full_nl_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_geography.yaml
new file mode 100644
index 000000000..de31a63b8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_nl_social_sciences_tasks
+task: global_mmlu_full_nl_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..bc0e3cb1b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_nl_social_sciences_tasks
+task: global_mmlu_full_nl_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..2e221c685
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_nl_social_sciences_tasks
+task: global_mmlu_full_nl_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_mathematics.yaml
new file mode 100644
index 000000000..137158a69
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_nl_stem_tasks
+task: global_mmlu_full_nl_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_microeconomics.yaml
new file mode 100644
index 000000000..27b426c06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_nl_social_sciences_tasks
+task: global_mmlu_full_nl_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_physics.yaml
new file mode 100644
index 000000000..746df49e8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_nl_stem_tasks
+task: global_mmlu_full_nl_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_psychology.yaml
new file mode 100644
index 000000000..89cb42d94
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_nl_social_sciences_tasks
+task: global_mmlu_full_nl_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_statistics.yaml
new file mode 100644
index 000000000..e27082c4c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_nl_stem_tasks
+task: global_mmlu_full_nl_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_us_history.yaml
new file mode 100644
index 000000000..66efc58c9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_nl_humanities_tasks
+task: global_mmlu_full_nl_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_world_history.yaml
new file mode 100644
index 000000000..83b653454
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_nl_humanities_tasks
+task: global_mmlu_full_nl_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_human_aging.yaml
new file mode 100644
index 000000000..82e00b4b8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_nl_other_tasks
+task: global_mmlu_full_nl_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_human_sexuality.yaml
new file mode 100644
index 000000000..468589da2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_nl_social_sciences_tasks
+task: global_mmlu_full_nl_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_international_law.yaml
new file mode 100644
index 000000000..e5bf62a98
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_nl_humanities_tasks
+task: global_mmlu_full_nl_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_jurisprudence.yaml
new file mode 100644
index 000000000..7b5336138
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_nl_humanities_tasks
+task: global_mmlu_full_nl_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_logical_fallacies.yaml
new file mode 100644
index 000000000..de862b663
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_nl_humanities_tasks
+task: global_mmlu_full_nl_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_machine_learning.yaml
new file mode 100644
index 000000000..c205af001
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_nl_stem_tasks
+task: global_mmlu_full_nl_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_management.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_management.yaml
new file mode 100644
index 000000000..5b624af8b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_nl_other_tasks
+task: global_mmlu_full_nl_management
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_marketing.yaml
new file mode 100644
index 000000000..81658e9f0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_nl_other_tasks
+task: global_mmlu_full_nl_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_medical_genetics.yaml
new file mode 100644
index 000000000..f8e52c0d4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_nl_other_tasks
+task: global_mmlu_full_nl_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_miscellaneous.yaml
new file mode 100644
index 000000000..31af482ec
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_nl_other_tasks
+task: global_mmlu_full_nl_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_moral_disputes.yaml
new file mode 100644
index 000000000..853de0c12
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_nl_humanities_tasks
+task: global_mmlu_full_nl_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_moral_scenarios.yaml
new file mode 100644
index 000000000..8b86e0450
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_nl_humanities_tasks
+task: global_mmlu_full_nl_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_nutrition.yaml
new file mode 100644
index 000000000..96036dae3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_nl_other_tasks
+task: global_mmlu_full_nl_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_philosophy.yaml
new file mode 100644
index 000000000..84e827dd1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_nl_humanities_tasks
+task: global_mmlu_full_nl_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_prehistory.yaml
new file mode 100644
index 000000000..f49c8a5e8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_nl_humanities_tasks
+task: global_mmlu_full_nl_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_accounting.yaml
new file mode 100644
index 000000000..45484116d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_nl_other_tasks
+task: global_mmlu_full_nl_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_law.yaml
new file mode 100644
index 000000000..17b28cd32
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_nl_humanities_tasks
+task: global_mmlu_full_nl_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_medicine.yaml
new file mode 100644
index 000000000..f4db01bc6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_nl_other_tasks
+task: global_mmlu_full_nl_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_psychology.yaml
new file mode 100644
index 000000000..be586b45d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_nl_social_sciences_tasks
+task: global_mmlu_full_nl_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_public_relations.yaml
new file mode 100644
index 000000000..2ffe58488
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_nl_social_sciences_tasks
+task: global_mmlu_full_nl_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_security_studies.yaml
new file mode 100644
index 000000000..b6c76948b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_nl_social_sciences_tasks
+task: global_mmlu_full_nl_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_sociology.yaml
new file mode 100644
index 000000000..983e13cde
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_nl_social_sciences_tasks
+task: global_mmlu_full_nl_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_us_foreign_policy.yaml
new file mode 100644
index 000000000..bd6b6227b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_nl_social_sciences_tasks
+task: global_mmlu_full_nl_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_virology.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_virology.yaml
new file mode 100644
index 000000000..92d1973bb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_nl_other_tasks
+task: global_mmlu_full_nl_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_world_religions.yaml
new file mode 100644
index 000000000..a8c2eccaa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _nl_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_nl_humanities_tasks
+task: global_mmlu_full_nl_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/nl/utils.py b/lmms_eval/tasks/global_mmlu/full/nl/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/nl/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny.yaml b/lmms_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny.yaml
new file mode 100644
index 000000000..80739c2c7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_ny
+task:
+  - global_mmlu_full_ny_stem
+  - global_mmlu_full_ny_other
+  - global_mmlu_full_ny_social_sciences
+  - global_mmlu_full_ny_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_humanities.yaml
new file mode 100644
index 000000000..89e7618fb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ny_humanities
+task:
+  - global_mmlu_full_ny_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_other.yaml b/lmms_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_other.yaml
new file mode 100644
index 000000000..51b90446d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ny_other
+task:
+  - global_mmlu_full_ny_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_social_sciences.yaml
new file mode 100644
index 000000000..b711dfdf1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ny_social_sciences
+task:
+  - global_mmlu_full_ny_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_stem.yaml b/lmms_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_stem.yaml
new file mode 100644
index 000000000..99bf9d95d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ny_stem
+task:
+  - global_mmlu_full_ny_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/_ny_template_yaml b/lmms_eval/tasks/global_mmlu/full/ny/_ny_template_yaml
new file mode 100644
index 000000000..069a9446e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/_ny_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: ny
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_abstract_algebra.yaml
new file mode 100644
index 000000000..2e3d7c33b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_ny_stem_tasks
+task: global_mmlu_full_ny_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_anatomy.yaml
new file mode 100644
index 000000000..60806afc6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_ny_stem_tasks
+task: global_mmlu_full_ny_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_astronomy.yaml
new file mode 100644
index 000000000..afbcb482f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_ny_stem_tasks
+task: global_mmlu_full_ny_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_business_ethics.yaml
new file mode 100644
index 000000000..6f8981bd0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_ny_other_tasks
+task: global_mmlu_full_ny_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_clinical_knowledge.yaml
new file mode 100644
index 000000000..ff44dd679
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_ny_other_tasks
+task: global_mmlu_full_ny_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_biology.yaml
new file mode 100644
index 000000000..da5ce3708
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_ny_stem_tasks
+task: global_mmlu_full_ny_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_chemistry.yaml
new file mode 100644
index 000000000..d62bce830
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_ny_stem_tasks
+task: global_mmlu_full_ny_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_computer_science.yaml
new file mode 100644
index 000000000..48cd98d55
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_ny_stem_tasks
+task: global_mmlu_full_ny_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_mathematics.yaml
new file mode 100644
index 000000000..ed77ba9c1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_ny_stem_tasks
+task: global_mmlu_full_ny_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_medicine.yaml
new file mode 100644
index 000000000..9cd8aa2f4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_ny_other_tasks
+task: global_mmlu_full_ny_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_physics.yaml
new file mode 100644
index 000000000..66d5dc27b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_ny_stem_tasks
+task: global_mmlu_full_ny_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_computer_security.yaml
new file mode 100644
index 000000000..8a9dae622
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_ny_stem_tasks
+task: global_mmlu_full_ny_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_conceptual_physics.yaml
new file mode 100644
index 000000000..8d160ffc6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_ny_stem_tasks
+task: global_mmlu_full_ny_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_econometrics.yaml
new file mode 100644
index 000000000..88af709a6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_ny_social_sciences_tasks
+task: global_mmlu_full_ny_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_electrical_engineering.yaml
new file mode 100644
index 000000000..d835f1e00
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_ny_stem_tasks
+task: global_mmlu_full_ny_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_elementary_mathematics.yaml
new file mode 100644
index 000000000..558ffd0ba
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_ny_stem_tasks
+task: global_mmlu_full_ny_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_formal_logic.yaml
new file mode 100644
index 000000000..cce0df192
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_ny_humanities_tasks
+task: global_mmlu_full_ny_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_global_facts.yaml
new file mode 100644
index 000000000..6ce027a55
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_ny_other_tasks
+task: global_mmlu_full_ny_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_biology.yaml
new file mode 100644
index 000000000..a729008d5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_ny_stem_tasks
+task: global_mmlu_full_ny_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_chemistry.yaml
new file mode 100644
index 000000000..79771bfb6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_ny_stem_tasks
+task: global_mmlu_full_ny_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_computer_science.yaml
new file mode 100644
index 000000000..6889806f2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_ny_stem_tasks
+task: global_mmlu_full_ny_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_european_history.yaml
new file mode 100644
index 000000000..29e6e4a5a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_ny_humanities_tasks
+task: global_mmlu_full_ny_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_geography.yaml
new file mode 100644
index 000000000..447db75fa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_ny_social_sciences_tasks
+task: global_mmlu_full_ny_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..e543cf760
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_ny_social_sciences_tasks
+task: global_mmlu_full_ny_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..61c49e750
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_ny_social_sciences_tasks
+task: global_mmlu_full_ny_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_mathematics.yaml
new file mode 100644
index 000000000..db228d021
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_ny_stem_tasks
+task: global_mmlu_full_ny_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_microeconomics.yaml
new file mode 100644
index 000000000..62d87c862
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_ny_social_sciences_tasks
+task: global_mmlu_full_ny_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_physics.yaml
new file mode 100644
index 000000000..54c15d66e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_ny_stem_tasks
+task: global_mmlu_full_ny_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_psychology.yaml
new file mode 100644
index 000000000..4f7d8b5a1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_ny_social_sciences_tasks
+task: global_mmlu_full_ny_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_statistics.yaml
new file mode 100644
index 000000000..f53235b87
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_ny_stem_tasks
+task: global_mmlu_full_ny_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_us_history.yaml
new file mode 100644
index 000000000..1d413b985
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_ny_humanities_tasks
+task: global_mmlu_full_ny_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_world_history.yaml
new file mode 100644
index 000000000..4adf2e8b1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_ny_humanities_tasks
+task: global_mmlu_full_ny_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_human_aging.yaml
new file mode 100644
index 000000000..9660b7b13
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_ny_other_tasks
+task: global_mmlu_full_ny_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_human_sexuality.yaml
new file mode 100644
index 000000000..11a6f2d4f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_ny_social_sciences_tasks
+task: global_mmlu_full_ny_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_international_law.yaml
new file mode 100644
index 000000000..9a46ff6a9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_ny_humanities_tasks
+task: global_mmlu_full_ny_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_jurisprudence.yaml
new file mode 100644
index 000000000..e4606df53
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_ny_humanities_tasks
+task: global_mmlu_full_ny_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_logical_fallacies.yaml
new file mode 100644
index 000000000..6edade038
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_ny_humanities_tasks
+task: global_mmlu_full_ny_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_machine_learning.yaml
new file mode 100644
index 000000000..765b22014
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_ny_stem_tasks
+task: global_mmlu_full_ny_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_management.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_management.yaml
new file mode 100644
index 000000000..a699a70db
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_ny_other_tasks
+task: global_mmlu_full_ny_management
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_marketing.yaml
new file mode 100644
index 000000000..596d69375
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_ny_other_tasks
+task: global_mmlu_full_ny_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_medical_genetics.yaml
new file mode 100644
index 000000000..4fae66a7c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_ny_other_tasks
+task: global_mmlu_full_ny_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_miscellaneous.yaml
new file mode 100644
index 000000000..8555e173a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_ny_other_tasks
+task: global_mmlu_full_ny_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_moral_disputes.yaml
new file mode 100644
index 000000000..b64f4d9da
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_ny_humanities_tasks
+task: global_mmlu_full_ny_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_moral_scenarios.yaml
new file mode 100644
index 000000000..c73f9f1af
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_ny_humanities_tasks
+task: global_mmlu_full_ny_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_nutrition.yaml
new file mode 100644
index 000000000..456f4cb63
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_ny_other_tasks
+task: global_mmlu_full_ny_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_philosophy.yaml
new file mode 100644
index 000000000..d0e0e05ed
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_ny_humanities_tasks
+task: global_mmlu_full_ny_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_prehistory.yaml
new file mode 100644
index 000000000..d65c6be1e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_ny_humanities_tasks
+task: global_mmlu_full_ny_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_accounting.yaml
new file mode 100644
index 000000000..c152c80ef
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_ny_other_tasks
+task: global_mmlu_full_ny_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_law.yaml
new file mode 100644
index 000000000..d5e2c7b7c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_ny_humanities_tasks
+task: global_mmlu_full_ny_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_medicine.yaml
new file mode 100644
index 000000000..cacd5df73
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_ny_other_tasks
+task: global_mmlu_full_ny_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_psychology.yaml
new file mode 100644
index 000000000..ffdd86d22
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_ny_social_sciences_tasks
+task: global_mmlu_full_ny_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_public_relations.yaml
new file mode 100644
index 000000000..0e6b5ab89
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_ny_social_sciences_tasks
+task: global_mmlu_full_ny_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_security_studies.yaml
new file mode 100644
index 000000000..f894fdd77
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_ny_social_sciences_tasks
+task: global_mmlu_full_ny_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_sociology.yaml
new file mode 100644
index 000000000..1d2d0cd4d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_ny_social_sciences_tasks
+task: global_mmlu_full_ny_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_us_foreign_policy.yaml
new file mode 100644
index 000000000..a72a237de
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_ny_social_sciences_tasks
+task: global_mmlu_full_ny_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_virology.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_virology.yaml
new file mode 100644
index 000000000..9eeb7cf03
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_ny_other_tasks
+task: global_mmlu_full_ny_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_world_religions.yaml
new file mode 100644
index 000000000..a1c243c83
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ny_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_ny_humanities_tasks
+task: global_mmlu_full_ny_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/ny/utils.py b/lmms_eval/tasks/global_mmlu/full/ny/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ny/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl.yaml b/lmms_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl.yaml
new file mode 100644
index 000000000..deb51e119
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_pl
+task:
+  - global_mmlu_full_pl_stem
+  - global_mmlu_full_pl_other
+  - global_mmlu_full_pl_social_sciences
+  - global_mmlu_full_pl_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_humanities.yaml
new file mode 100644
index 000000000..4b5f7aa45
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_pl_humanities
+task:
+  - global_mmlu_full_pl_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_other.yaml b/lmms_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_other.yaml
new file mode 100644
index 000000000..241dbc1ce
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_pl_other
+task:
+  - global_mmlu_full_pl_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_social_sciences.yaml
new file mode 100644
index 000000000..9a50a3152
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_pl_social_sciences
+task:
+  - global_mmlu_full_pl_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_stem.yaml b/lmms_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_stem.yaml
new file mode 100644
index 000000000..3d11c89f1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_pl_stem
+task:
+  - global_mmlu_full_pl_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/_pl_template_yaml b/lmms_eval/tasks/global_mmlu/full/pl/_pl_template_yaml
new file mode 100644
index 000000000..af8809dc2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/_pl_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: pl
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_abstract_algebra.yaml
new file mode 100644
index 000000000..37f611a1b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_pl_stem_tasks
+task: global_mmlu_full_pl_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_anatomy.yaml
new file mode 100644
index 000000000..c274bce17
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_pl_stem_tasks
+task: global_mmlu_full_pl_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_astronomy.yaml
new file mode 100644
index 000000000..99220f0dd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_pl_stem_tasks
+task: global_mmlu_full_pl_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_business_ethics.yaml
new file mode 100644
index 000000000..105926680
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_pl_other_tasks
+task: global_mmlu_full_pl_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_clinical_knowledge.yaml
new file mode 100644
index 000000000..29a4fadc0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_pl_other_tasks
+task: global_mmlu_full_pl_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_biology.yaml
new file mode 100644
index 000000000..cce1671ce
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_pl_stem_tasks
+task: global_mmlu_full_pl_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_chemistry.yaml
new file mode 100644
index 000000000..79c635308
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_pl_stem_tasks
+task: global_mmlu_full_pl_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_computer_science.yaml
new file mode 100644
index 000000000..bb630140e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_pl_stem_tasks
+task: global_mmlu_full_pl_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_mathematics.yaml
new file mode 100644
index 000000000..6b42f767e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_pl_stem_tasks
+task: global_mmlu_full_pl_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_medicine.yaml
new file mode 100644
index 000000000..43bea9764
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_pl_other_tasks
+task: global_mmlu_full_pl_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_physics.yaml
new file mode 100644
index 000000000..0c9ea6014
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_pl_stem_tasks
+task: global_mmlu_full_pl_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_computer_security.yaml
new file mode 100644
index 000000000..365b60a3e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_pl_stem_tasks
+task: global_mmlu_full_pl_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_conceptual_physics.yaml
new file mode 100644
index 000000000..2b9437e3f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_pl_stem_tasks
+task: global_mmlu_full_pl_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_econometrics.yaml
new file mode 100644
index 000000000..648f24c42
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_pl_social_sciences_tasks
+task: global_mmlu_full_pl_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_electrical_engineering.yaml
new file mode 100644
index 000000000..196de258d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_pl_stem_tasks
+task: global_mmlu_full_pl_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_elementary_mathematics.yaml
new file mode 100644
index 000000000..8646b6a39
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_pl_stem_tasks
+task: global_mmlu_full_pl_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_formal_logic.yaml
new file mode 100644
index 000000000..2d13d283c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_pl_humanities_tasks
+task: global_mmlu_full_pl_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_global_facts.yaml
new file mode 100644
index 000000000..15bb640b9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_pl_other_tasks
+task: global_mmlu_full_pl_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_biology.yaml
new file mode 100644
index 000000000..ba9640283
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_pl_stem_tasks
+task: global_mmlu_full_pl_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_chemistry.yaml
new file mode 100644
index 000000000..7f142dd82
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_pl_stem_tasks
+task: global_mmlu_full_pl_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_computer_science.yaml
new file mode 100644
index 000000000..99b3b9da1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_pl_stem_tasks
+task: global_mmlu_full_pl_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_european_history.yaml
new file mode 100644
index 000000000..e99b2fb93
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_pl_humanities_tasks
+task: global_mmlu_full_pl_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_geography.yaml
new file mode 100644
index 000000000..bc6113f78
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_pl_social_sciences_tasks
+task: global_mmlu_full_pl_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..05a7de9bf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_pl_social_sciences_tasks
+task: global_mmlu_full_pl_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..aceda633f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_pl_social_sciences_tasks
+task: global_mmlu_full_pl_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_mathematics.yaml
new file mode 100644
index 000000000..6eef2cd85
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_pl_stem_tasks
+task: global_mmlu_full_pl_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_microeconomics.yaml
new file mode 100644
index 000000000..5adb5fa19
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_pl_social_sciences_tasks
+task: global_mmlu_full_pl_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_physics.yaml
new file mode 100644
index 000000000..fbda79209
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_pl_stem_tasks
+task: global_mmlu_full_pl_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_psychology.yaml
new file mode 100644
index 000000000..7eb09362a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_pl_social_sciences_tasks
+task: global_mmlu_full_pl_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_statistics.yaml
new file mode 100644
index 000000000..b7beef5b0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_pl_stem_tasks
+task: global_mmlu_full_pl_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_us_history.yaml
new file mode 100644
index 000000000..08f45dd9c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_pl_humanities_tasks
+task: global_mmlu_full_pl_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_world_history.yaml
new file mode 100644
index 000000000..99664de87
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_pl_humanities_tasks
+task: global_mmlu_full_pl_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_human_aging.yaml
new file mode 100644
index 000000000..d63f6f8d4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_pl_other_tasks
+task: global_mmlu_full_pl_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_human_sexuality.yaml
new file mode 100644
index 000000000..8080ca8d0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_pl_social_sciences_tasks
+task: global_mmlu_full_pl_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_international_law.yaml
new file mode 100644
index 000000000..425695c1d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_pl_humanities_tasks
+task: global_mmlu_full_pl_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_jurisprudence.yaml
new file mode 100644
index 000000000..a6455bd71
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_pl_humanities_tasks
+task: global_mmlu_full_pl_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_logical_fallacies.yaml
new file mode 100644
index 000000000..f1359b3a2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_pl_humanities_tasks
+task: global_mmlu_full_pl_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_machine_learning.yaml
new file mode 100644
index 000000000..3d7bb0dcc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_pl_stem_tasks
+task: global_mmlu_full_pl_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_management.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_management.yaml
new file mode 100644
index 000000000..f695226c1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_pl_other_tasks
+task: global_mmlu_full_pl_management
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_marketing.yaml
new file mode 100644
index 000000000..7fedcd3f8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_pl_other_tasks
+task: global_mmlu_full_pl_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_medical_genetics.yaml
new file mode 100644
index 000000000..89da9f675
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_pl_other_tasks
+task: global_mmlu_full_pl_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_miscellaneous.yaml
new file mode 100644
index 000000000..6f34762c9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_pl_other_tasks
+task: global_mmlu_full_pl_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_moral_disputes.yaml
new file mode 100644
index 000000000..25f201f4b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_pl_humanities_tasks
+task: global_mmlu_full_pl_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_moral_scenarios.yaml
new file mode 100644
index 000000000..fd08e6e13
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_pl_humanities_tasks
+task: global_mmlu_full_pl_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_nutrition.yaml
new file mode 100644
index 000000000..b61f1f17f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_pl_other_tasks
+task: global_mmlu_full_pl_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_philosophy.yaml
new file mode 100644
index 000000000..8c1bf6dc1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_pl_humanities_tasks
+task: global_mmlu_full_pl_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_prehistory.yaml
new file mode 100644
index 000000000..e5329e133
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_pl_humanities_tasks
+task: global_mmlu_full_pl_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_accounting.yaml
new file mode 100644
index 000000000..514b04cd1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_pl_other_tasks
+task: global_mmlu_full_pl_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_law.yaml
new file mode 100644
index 000000000..99c719f83
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_pl_humanities_tasks
+task: global_mmlu_full_pl_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_medicine.yaml
new file mode 100644
index 000000000..1dfafb254
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_pl_other_tasks
+task: global_mmlu_full_pl_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_psychology.yaml
new file mode 100644
index 000000000..5b6181c0c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_pl_social_sciences_tasks
+task: global_mmlu_full_pl_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_public_relations.yaml
new file mode 100644
index 000000000..acf874dbf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_pl_social_sciences_tasks
+task: global_mmlu_full_pl_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_security_studies.yaml
new file mode 100644
index 000000000..d754904c4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_pl_social_sciences_tasks
+task: global_mmlu_full_pl_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_sociology.yaml
new file mode 100644
index 000000000..4bc0fd8ff
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_pl_social_sciences_tasks
+task: global_mmlu_full_pl_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_us_foreign_policy.yaml
new file mode 100644
index 000000000..ef719be1f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_pl_social_sciences_tasks
+task: global_mmlu_full_pl_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_virology.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_virology.yaml
new file mode 100644
index 000000000..f9084c13d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_pl_other_tasks
+task: global_mmlu_full_pl_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_world_religions.yaml
new file mode 100644
index 000000000..036d0f4c0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pl_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_pl_humanities_tasks
+task: global_mmlu_full_pl_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/pl/utils.py b/lmms_eval/tasks/global_mmlu/full/pl/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pl/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt.yaml b/lmms_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt.yaml
new file mode 100644
index 000000000..7cb9f0fe3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_pt
+task:
+  - global_mmlu_full_pt_stem
+  - global_mmlu_full_pt_other
+  - global_mmlu_full_pt_social_sciences
+  - global_mmlu_full_pt_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_humanities.yaml
new file mode 100644
index 000000000..261a70288
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_pt_humanities
+task:
+  - global_mmlu_full_pt_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_other.yaml b/lmms_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_other.yaml
new file mode 100644
index 000000000..a61b12f50
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_pt_other
+task:
+  - global_mmlu_full_pt_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_social_sciences.yaml
new file mode 100644
index 000000000..2c04bf5a4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_pt_social_sciences
+task:
+  - global_mmlu_full_pt_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_stem.yaml b/lmms_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_stem.yaml
new file mode 100644
index 000000000..dc3d36104
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_pt_stem
+task:
+  - global_mmlu_full_pt_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/_pt_template_yaml b/lmms_eval/tasks/global_mmlu/full/pt/_pt_template_yaml
new file mode 100644
index 000000000..66ba2417d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/_pt_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: pt
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_abstract_algebra.yaml
new file mode 100644
index 000000000..d9efd8170
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_pt_stem_tasks
+task: global_mmlu_full_pt_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_anatomy.yaml
new file mode 100644
index 000000000..45390503d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_pt_stem_tasks
+task: global_mmlu_full_pt_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_astronomy.yaml
new file mode 100644
index 000000000..90880cd07
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_pt_stem_tasks
+task: global_mmlu_full_pt_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_business_ethics.yaml
new file mode 100644
index 000000000..f18ef2d85
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_pt_other_tasks
+task: global_mmlu_full_pt_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_clinical_knowledge.yaml
new file mode 100644
index 000000000..2999a02a7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_pt_other_tasks
+task: global_mmlu_full_pt_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_biology.yaml
new file mode 100644
index 000000000..0cf0a61b5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_pt_stem_tasks
+task: global_mmlu_full_pt_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_chemistry.yaml
new file mode 100644
index 000000000..91d8cd2ed
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_pt_stem_tasks
+task: global_mmlu_full_pt_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_computer_science.yaml
new file mode 100644
index 000000000..68592aafe
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_pt_stem_tasks
+task: global_mmlu_full_pt_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_mathematics.yaml
new file mode 100644
index 000000000..31d7f6afd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_pt_stem_tasks
+task: global_mmlu_full_pt_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_medicine.yaml
new file mode 100644
index 000000000..46ec82324
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_pt_other_tasks
+task: global_mmlu_full_pt_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_physics.yaml
new file mode 100644
index 000000000..2cf6402dd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_pt_stem_tasks
+task: global_mmlu_full_pt_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_computer_security.yaml
new file mode 100644
index 000000000..0953a105a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_pt_stem_tasks
+task: global_mmlu_full_pt_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_conceptual_physics.yaml
new file mode 100644
index 000000000..0e6e91a99
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_pt_stem_tasks
+task: global_mmlu_full_pt_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_econometrics.yaml
new file mode 100644
index 000000000..67c29915c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_pt_social_sciences_tasks
+task: global_mmlu_full_pt_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_electrical_engineering.yaml
new file mode 100644
index 000000000..5a6ba82e2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_pt_stem_tasks
+task: global_mmlu_full_pt_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_elementary_mathematics.yaml
new file mode 100644
index 000000000..3d66a664c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_pt_stem_tasks
+task: global_mmlu_full_pt_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_formal_logic.yaml
new file mode 100644
index 000000000..683d6dddc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_pt_humanities_tasks
+task: global_mmlu_full_pt_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_global_facts.yaml
new file mode 100644
index 000000000..e4396542a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_pt_other_tasks
+task: global_mmlu_full_pt_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_biology.yaml
new file mode 100644
index 000000000..89fefd1c3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_pt_stem_tasks
+task: global_mmlu_full_pt_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_chemistry.yaml
new file mode 100644
index 000000000..ea323d8a6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_pt_stem_tasks
+task: global_mmlu_full_pt_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_computer_science.yaml
new file mode 100644
index 000000000..5f8f00823
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_pt_stem_tasks
+task: global_mmlu_full_pt_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_european_history.yaml
new file mode 100644
index 000000000..bef7a3168
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_pt_humanities_tasks
+task: global_mmlu_full_pt_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_geography.yaml
new file mode 100644
index 000000000..e69c29780
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_pt_social_sciences_tasks
+task: global_mmlu_full_pt_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..e3fa920d7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_pt_social_sciences_tasks
+task: global_mmlu_full_pt_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..6b7ca2f4e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_pt_social_sciences_tasks
+task: global_mmlu_full_pt_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_mathematics.yaml
new file mode 100644
index 000000000..4713674dc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_pt_stem_tasks
+task: global_mmlu_full_pt_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_microeconomics.yaml
new file mode 100644
index 000000000..d6475e995
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_pt_social_sciences_tasks
+task: global_mmlu_full_pt_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_physics.yaml
new file mode 100644
index 000000000..9eaed31ad
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_pt_stem_tasks
+task: global_mmlu_full_pt_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_psychology.yaml
new file mode 100644
index 000000000..d09e1eb97
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_pt_social_sciences_tasks
+task: global_mmlu_full_pt_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_statistics.yaml
new file mode 100644
index 000000000..3d8c1447f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_pt_stem_tasks
+task: global_mmlu_full_pt_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_us_history.yaml
new file mode 100644
index 000000000..a883b438e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_pt_humanities_tasks
+task: global_mmlu_full_pt_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_world_history.yaml
new file mode 100644
index 000000000..6ea1454e3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_pt_humanities_tasks
+task: global_mmlu_full_pt_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_human_aging.yaml
new file mode 100644
index 000000000..34033c55b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_pt_other_tasks
+task: global_mmlu_full_pt_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_human_sexuality.yaml
new file mode 100644
index 000000000..bf961c33a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_pt_social_sciences_tasks
+task: global_mmlu_full_pt_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_international_law.yaml
new file mode 100644
index 000000000..5247fc9bd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_pt_humanities_tasks
+task: global_mmlu_full_pt_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_jurisprudence.yaml
new file mode 100644
index 000000000..07e78da50
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_pt_humanities_tasks
+task: global_mmlu_full_pt_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_logical_fallacies.yaml
new file mode 100644
index 000000000..c2451399b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_pt_humanities_tasks
+task: global_mmlu_full_pt_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_machine_learning.yaml
new file mode 100644
index 000000000..79c577eac
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_pt_stem_tasks
+task: global_mmlu_full_pt_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_management.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_management.yaml
new file mode 100644
index 000000000..a344b1c32
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_pt_other_tasks
+task: global_mmlu_full_pt_management
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_marketing.yaml
new file mode 100644
index 000000000..eeff36b97
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_pt_other_tasks
+task: global_mmlu_full_pt_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_medical_genetics.yaml
new file mode 100644
index 000000000..27985380f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_pt_other_tasks
+task: global_mmlu_full_pt_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_miscellaneous.yaml
new file mode 100644
index 000000000..e2fa1da1c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_pt_other_tasks
+task: global_mmlu_full_pt_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_moral_disputes.yaml
new file mode 100644
index 000000000..e83d186e2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_pt_humanities_tasks
+task: global_mmlu_full_pt_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_moral_scenarios.yaml
new file mode 100644
index 000000000..3529a15c3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_pt_humanities_tasks
+task: global_mmlu_full_pt_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_nutrition.yaml
new file mode 100644
index 000000000..e51eefe02
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_pt_other_tasks
+task: global_mmlu_full_pt_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_philosophy.yaml
new file mode 100644
index 000000000..ec0826b95
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_pt_humanities_tasks
+task: global_mmlu_full_pt_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_prehistory.yaml
new file mode 100644
index 000000000..324dfe69e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_pt_humanities_tasks
+task: global_mmlu_full_pt_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_accounting.yaml
new file mode 100644
index 000000000..530c918ef
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_pt_other_tasks
+task: global_mmlu_full_pt_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_law.yaml
new file mode 100644
index 000000000..f7a3679c9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_pt_humanities_tasks
+task: global_mmlu_full_pt_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_medicine.yaml
new file mode 100644
index 000000000..0f4cc0066
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_pt_other_tasks
+task: global_mmlu_full_pt_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_psychology.yaml
new file mode 100644
index 000000000..4c5884c9f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_pt_social_sciences_tasks
+task: global_mmlu_full_pt_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_public_relations.yaml
new file mode 100644
index 000000000..bb2d65360
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_pt_social_sciences_tasks
+task: global_mmlu_full_pt_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_security_studies.yaml
new file mode 100644
index 000000000..1af8d6621
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_pt_social_sciences_tasks
+task: global_mmlu_full_pt_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_sociology.yaml
new file mode 100644
index 000000000..3ef8fcb79
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_pt_social_sciences_tasks
+task: global_mmlu_full_pt_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_us_foreign_policy.yaml
new file mode 100644
index 000000000..8b48f5287
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_pt_social_sciences_tasks
+task: global_mmlu_full_pt_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_virology.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_virology.yaml
new file mode 100644
index 000000000..4b0de7532
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_pt_other_tasks
+task: global_mmlu_full_pt_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_world_religions.yaml
new file mode 100644
index 000000000..796485864
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _pt_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_pt_humanities_tasks
+task: global_mmlu_full_pt_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/pt/utils.py b/lmms_eval/tasks/global_mmlu/full/pt/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/pt/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro.yaml b/lmms_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro.yaml
new file mode 100644
index 000000000..f4e5575f0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_ro
+task:
+  - global_mmlu_full_ro_stem
+  - global_mmlu_full_ro_other
+  - global_mmlu_full_ro_social_sciences
+  - global_mmlu_full_ro_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_humanities.yaml
new file mode 100644
index 000000000..d54268b0a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ro_humanities
+task:
+  - global_mmlu_full_ro_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_other.yaml b/lmms_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_other.yaml
new file mode 100644
index 000000000..4e58aea9a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ro_other
+task:
+  - global_mmlu_full_ro_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_social_sciences.yaml
new file mode 100644
index 000000000..e1cb84a91
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ro_social_sciences
+task:
+  - global_mmlu_full_ro_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_stem.yaml b/lmms_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_stem.yaml
new file mode 100644
index 000000000..de0e406fa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ro_stem
+task:
+  - global_mmlu_full_ro_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/_ro_template_yaml b/lmms_eval/tasks/global_mmlu/full/ro/_ro_template_yaml
new file mode 100644
index 000000000..e5cb6dd0e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/_ro_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: ro
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_abstract_algebra.yaml
new file mode 100644
index 000000000..c505fb8b2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_ro_stem_tasks
+task: global_mmlu_full_ro_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_anatomy.yaml
new file mode 100644
index 000000000..0c13018cd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_ro_stem_tasks
+task: global_mmlu_full_ro_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_astronomy.yaml
new file mode 100644
index 000000000..9f4caefba
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_ro_stem_tasks
+task: global_mmlu_full_ro_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_business_ethics.yaml
new file mode 100644
index 000000000..1c1387fd1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_ro_other_tasks
+task: global_mmlu_full_ro_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_clinical_knowledge.yaml
new file mode 100644
index 000000000..b9e0dbb47
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_ro_other_tasks
+task: global_mmlu_full_ro_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_biology.yaml
new file mode 100644
index 000000000..5bf14ab0f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_ro_stem_tasks
+task: global_mmlu_full_ro_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_chemistry.yaml
new file mode 100644
index 000000000..590347443
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_ro_stem_tasks
+task: global_mmlu_full_ro_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_computer_science.yaml
new file mode 100644
index 000000000..6bb64c2e2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_ro_stem_tasks
+task: global_mmlu_full_ro_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_mathematics.yaml
new file mode 100644
index 000000000..d719a5ef9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_ro_stem_tasks
+task: global_mmlu_full_ro_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_medicine.yaml
new file mode 100644
index 000000000..c9284a8f9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_ro_other_tasks
+task: global_mmlu_full_ro_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_physics.yaml
new file mode 100644
index 000000000..1d27d8437
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_ro_stem_tasks
+task: global_mmlu_full_ro_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_computer_security.yaml
new file mode 100644
index 000000000..1d63556e3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_ro_stem_tasks
+task: global_mmlu_full_ro_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_conceptual_physics.yaml
new file mode 100644
index 000000000..25f30a368
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_ro_stem_tasks
+task: global_mmlu_full_ro_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_econometrics.yaml
new file mode 100644
index 000000000..1fa6b5d23
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_ro_social_sciences_tasks
+task: global_mmlu_full_ro_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_electrical_engineering.yaml
new file mode 100644
index 000000000..f6eb4b6e4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_ro_stem_tasks
+task: global_mmlu_full_ro_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_elementary_mathematics.yaml
new file mode 100644
index 000000000..e99772e27
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_ro_stem_tasks
+task: global_mmlu_full_ro_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_formal_logic.yaml
new file mode 100644
index 000000000..be99bd007
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_ro_humanities_tasks
+task: global_mmlu_full_ro_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_global_facts.yaml
new file mode 100644
index 000000000..819937e79
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_ro_other_tasks
+task: global_mmlu_full_ro_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_biology.yaml
new file mode 100644
index 000000000..d7509581c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_ro_stem_tasks
+task: global_mmlu_full_ro_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_chemistry.yaml
new file mode 100644
index 000000000..d089583f6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_ro_stem_tasks
+task: global_mmlu_full_ro_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_computer_science.yaml
new file mode 100644
index 000000000..46d5f4723
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_ro_stem_tasks
+task: global_mmlu_full_ro_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_european_history.yaml
new file mode 100644
index 000000000..1a1ae7e72
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_ro_humanities_tasks
+task: global_mmlu_full_ro_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_geography.yaml
new file mode 100644
index 000000000..92935be5c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_ro_social_sciences_tasks
+task: global_mmlu_full_ro_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..efd2a03fa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_ro_social_sciences_tasks
+task: global_mmlu_full_ro_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..fe2f97d1f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_ro_social_sciences_tasks
+task: global_mmlu_full_ro_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_mathematics.yaml
new file mode 100644
index 000000000..f0432a01f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_ro_stem_tasks
+task: global_mmlu_full_ro_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_microeconomics.yaml
new file mode 100644
index 000000000..507fab869
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_ro_social_sciences_tasks
+task: global_mmlu_full_ro_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_physics.yaml
new file mode 100644
index 000000000..19a76707e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_ro_stem_tasks
+task: global_mmlu_full_ro_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_psychology.yaml
new file mode 100644
index 000000000..d27fc262e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_ro_social_sciences_tasks
+task: global_mmlu_full_ro_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_statistics.yaml
new file mode 100644
index 000000000..8f8023bcf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_ro_stem_tasks
+task: global_mmlu_full_ro_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_us_history.yaml
new file mode 100644
index 000000000..acc5fc418
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_ro_humanities_tasks
+task: global_mmlu_full_ro_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_world_history.yaml
new file mode 100644
index 000000000..9ea7c9336
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_ro_humanities_tasks
+task: global_mmlu_full_ro_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_human_aging.yaml
new file mode 100644
index 000000000..6b984c55b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_ro_other_tasks
+task: global_mmlu_full_ro_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_human_sexuality.yaml
new file mode 100644
index 000000000..e2af2cbee
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_ro_social_sciences_tasks
+task: global_mmlu_full_ro_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_international_law.yaml
new file mode 100644
index 000000000..1cbf3d032
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_ro_humanities_tasks
+task: global_mmlu_full_ro_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_jurisprudence.yaml
new file mode 100644
index 000000000..d0acaca0a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_ro_humanities_tasks
+task: global_mmlu_full_ro_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_logical_fallacies.yaml
new file mode 100644
index 000000000..c84234a0b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_ro_humanities_tasks
+task: global_mmlu_full_ro_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_machine_learning.yaml
new file mode 100644
index 000000000..09237c9ee
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_ro_stem_tasks
+task: global_mmlu_full_ro_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_management.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_management.yaml
new file mode 100644
index 000000000..fcb3f4853
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_ro_other_tasks
+task: global_mmlu_full_ro_management
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_marketing.yaml
new file mode 100644
index 000000000..33b486c04
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_ro_other_tasks
+task: global_mmlu_full_ro_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_medical_genetics.yaml
new file mode 100644
index 000000000..09c3d5e92
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_ro_other_tasks
+task: global_mmlu_full_ro_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_miscellaneous.yaml
new file mode 100644
index 000000000..e744e1e7c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_ro_other_tasks
+task: global_mmlu_full_ro_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_moral_disputes.yaml
new file mode 100644
index 000000000..4e6d4ed7e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_ro_humanities_tasks
+task: global_mmlu_full_ro_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_moral_scenarios.yaml
new file mode 100644
index 000000000..d0e99149a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_ro_humanities_tasks
+task: global_mmlu_full_ro_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_nutrition.yaml
new file mode 100644
index 000000000..850262c1e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_ro_other_tasks
+task: global_mmlu_full_ro_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_philosophy.yaml
new file mode 100644
index 000000000..9dd2bf54a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_ro_humanities_tasks
+task: global_mmlu_full_ro_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_prehistory.yaml
new file mode 100644
index 000000000..b2ecf40d7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_ro_humanities_tasks
+task: global_mmlu_full_ro_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_accounting.yaml
new file mode 100644
index 000000000..db2597667
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_ro_other_tasks
+task: global_mmlu_full_ro_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_law.yaml
new file mode 100644
index 000000000..b1e439748
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_ro_humanities_tasks
+task: global_mmlu_full_ro_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_medicine.yaml
new file mode 100644
index 000000000..0158c5453
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_ro_other_tasks
+task: global_mmlu_full_ro_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_psychology.yaml
new file mode 100644
index 000000000..bdd7ca7f2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_ro_social_sciences_tasks
+task: global_mmlu_full_ro_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_public_relations.yaml
new file mode 100644
index 000000000..5f7f0f514
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_ro_social_sciences_tasks
+task: global_mmlu_full_ro_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_security_studies.yaml
new file mode 100644
index 000000000..be9b334ec
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_ro_social_sciences_tasks
+task: global_mmlu_full_ro_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_sociology.yaml
new file mode 100644
index 000000000..f37228bde
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_ro_social_sciences_tasks
+task: global_mmlu_full_ro_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_us_foreign_policy.yaml
new file mode 100644
index 000000000..aae05dc98
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_ro_social_sciences_tasks
+task: global_mmlu_full_ro_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_virology.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_virology.yaml
new file mode 100644
index 000000000..2d789c204
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_ro_other_tasks
+task: global_mmlu_full_ro_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_world_religions.yaml
new file mode 100644
index 000000000..40ff82284
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ro_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_ro_humanities_tasks
+task: global_mmlu_full_ro_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/ro/utils.py b/lmms_eval/tasks/global_mmlu/full/ro/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ro/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru.yaml b/lmms_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru.yaml
new file mode 100644
index 000000000..3fc05379c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_ru
+task:
+  - global_mmlu_full_ru_stem
+  - global_mmlu_full_ru_other
+  - global_mmlu_full_ru_social_sciences
+  - global_mmlu_full_ru_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_humanities.yaml
new file mode 100644
index 000000000..55422b437
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ru_humanities
+task:
+  - global_mmlu_full_ru_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_other.yaml b/lmms_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_other.yaml
new file mode 100644
index 000000000..d47ccc605
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ru_other
+task:
+  - global_mmlu_full_ru_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_social_sciences.yaml
new file mode 100644
index 000000000..12d48428a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ru_social_sciences
+task:
+  - global_mmlu_full_ru_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_stem.yaml b/lmms_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_stem.yaml
new file mode 100644
index 000000000..70ae3edb7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_ru_stem
+task:
+  - global_mmlu_full_ru_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/_ru_template_yaml b/lmms_eval/tasks/global_mmlu/full/ru/_ru_template_yaml
new file mode 100644
index 000000000..4b2f491bd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/_ru_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: ru
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_abstract_algebra.yaml
new file mode 100644
index 000000000..de158df87
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_ru_stem_tasks
+task: global_mmlu_full_ru_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_anatomy.yaml
new file mode 100644
index 000000000..aab717e4f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_ru_stem_tasks
+task: global_mmlu_full_ru_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_astronomy.yaml
new file mode 100644
index 000000000..3d8d0e32e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_ru_stem_tasks
+task: global_mmlu_full_ru_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_business_ethics.yaml
new file mode 100644
index 000000000..d2855ca36
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_ru_other_tasks
+task: global_mmlu_full_ru_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_clinical_knowledge.yaml
new file mode 100644
index 000000000..2efe08298
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_ru_other_tasks
+task: global_mmlu_full_ru_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_biology.yaml
new file mode 100644
index 000000000..96d00deb8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_ru_stem_tasks
+task: global_mmlu_full_ru_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_chemistry.yaml
new file mode 100644
index 000000000..0a5aac35c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_ru_stem_tasks
+task: global_mmlu_full_ru_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_computer_science.yaml
new file mode 100644
index 000000000..bd8bf28bd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_ru_stem_tasks
+task: global_mmlu_full_ru_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_mathematics.yaml
new file mode 100644
index 000000000..a2e080c3d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_ru_stem_tasks
+task: global_mmlu_full_ru_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_medicine.yaml
new file mode 100644
index 000000000..70e8448ea
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_ru_other_tasks
+task: global_mmlu_full_ru_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_physics.yaml
new file mode 100644
index 000000000..8e6ecbcf2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_ru_stem_tasks
+task: global_mmlu_full_ru_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_computer_security.yaml
new file mode 100644
index 000000000..f196351af
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_ru_stem_tasks
+task: global_mmlu_full_ru_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_conceptual_physics.yaml
new file mode 100644
index 000000000..e623d78fd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_ru_stem_tasks
+task: global_mmlu_full_ru_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_econometrics.yaml
new file mode 100644
index 000000000..df35a1f60
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_ru_social_sciences_tasks
+task: global_mmlu_full_ru_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_electrical_engineering.yaml
new file mode 100644
index 000000000..82c49f89e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_ru_stem_tasks
+task: global_mmlu_full_ru_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_elementary_mathematics.yaml
new file mode 100644
index 000000000..6ed11c5fa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_ru_stem_tasks
+task: global_mmlu_full_ru_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_formal_logic.yaml
new file mode 100644
index 000000000..8ebe62bf4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_ru_humanities_tasks
+task: global_mmlu_full_ru_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_global_facts.yaml
new file mode 100644
index 000000000..27d6ad700
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_ru_other_tasks
+task: global_mmlu_full_ru_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_biology.yaml
new file mode 100644
index 000000000..7860e73e7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_ru_stem_tasks
+task: global_mmlu_full_ru_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_chemistry.yaml
new file mode 100644
index 000000000..7596daa3b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_ru_stem_tasks
+task: global_mmlu_full_ru_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_computer_science.yaml
new file mode 100644
index 000000000..ecb64d52c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_ru_stem_tasks
+task: global_mmlu_full_ru_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_european_history.yaml
new file mode 100644
index 000000000..92feccc59
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_ru_humanities_tasks
+task: global_mmlu_full_ru_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_geography.yaml
new file mode 100644
index 000000000..6f586f500
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_ru_social_sciences_tasks
+task: global_mmlu_full_ru_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..0ffc85dfe
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_ru_social_sciences_tasks
+task: global_mmlu_full_ru_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..5da132047
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_ru_social_sciences_tasks
+task: global_mmlu_full_ru_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_mathematics.yaml
new file mode 100644
index 000000000..fc684975c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_ru_stem_tasks
+task: global_mmlu_full_ru_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_microeconomics.yaml
new file mode 100644
index 000000000..84887d18a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_ru_social_sciences_tasks
+task: global_mmlu_full_ru_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_physics.yaml
new file mode 100644
index 000000000..29ddf5bf3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_ru_stem_tasks
+task: global_mmlu_full_ru_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_psychology.yaml
new file mode 100644
index 000000000..a0680bad6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_ru_social_sciences_tasks
+task: global_mmlu_full_ru_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_statistics.yaml
new file mode 100644
index 000000000..07ac341bf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_ru_stem_tasks
+task: global_mmlu_full_ru_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_us_history.yaml
new file mode 100644
index 000000000..18e12bcd3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_ru_humanities_tasks
+task: global_mmlu_full_ru_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_world_history.yaml
new file mode 100644
index 000000000..c37522a22
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_ru_humanities_tasks
+task: global_mmlu_full_ru_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_human_aging.yaml
new file mode 100644
index 000000000..cbd6bf329
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_ru_other_tasks
+task: global_mmlu_full_ru_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_human_sexuality.yaml
new file mode 100644
index 000000000..8766c3489
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_ru_social_sciences_tasks
+task: global_mmlu_full_ru_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_international_law.yaml
new file mode 100644
index 000000000..4edbb98cf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_ru_humanities_tasks
+task: global_mmlu_full_ru_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_jurisprudence.yaml
new file mode 100644
index 000000000..24cea6320
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_ru_humanities_tasks
+task: global_mmlu_full_ru_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_logical_fallacies.yaml
new file mode 100644
index 000000000..3160fadc3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_ru_humanities_tasks
+task: global_mmlu_full_ru_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_machine_learning.yaml
new file mode 100644
index 000000000..b8e480e68
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_ru_stem_tasks
+task: global_mmlu_full_ru_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_management.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_management.yaml
new file mode 100644
index 000000000..4a7b77a14
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_ru_other_tasks
+task: global_mmlu_full_ru_management
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_marketing.yaml
new file mode 100644
index 000000000..c71a4f292
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_ru_other_tasks
+task: global_mmlu_full_ru_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_medical_genetics.yaml
new file mode 100644
index 000000000..ac34ba203
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_ru_other_tasks
+task: global_mmlu_full_ru_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_miscellaneous.yaml
new file mode 100644
index 000000000..6049ccb1f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_ru_other_tasks
+task: global_mmlu_full_ru_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_moral_disputes.yaml
new file mode 100644
index 000000000..d974ccfaa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_ru_humanities_tasks
+task: global_mmlu_full_ru_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_moral_scenarios.yaml
new file mode 100644
index 000000000..f05f7de9a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_ru_humanities_tasks
+task: global_mmlu_full_ru_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_nutrition.yaml
new file mode 100644
index 000000000..59cc8deed
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_ru_other_tasks
+task: global_mmlu_full_ru_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_philosophy.yaml
new file mode 100644
index 000000000..eb78b1f75
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_ru_humanities_tasks
+task: global_mmlu_full_ru_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_prehistory.yaml
new file mode 100644
index 000000000..685bb2a4d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_ru_humanities_tasks
+task: global_mmlu_full_ru_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_accounting.yaml
new file mode 100644
index 000000000..35c21255f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_ru_other_tasks
+task: global_mmlu_full_ru_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_law.yaml
new file mode 100644
index 000000000..ce70d0067
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_ru_humanities_tasks
+task: global_mmlu_full_ru_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_medicine.yaml
new file mode 100644
index 000000000..cce88d1dd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_ru_other_tasks
+task: global_mmlu_full_ru_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_psychology.yaml
new file mode 100644
index 000000000..39fc89538
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_ru_social_sciences_tasks
+task: global_mmlu_full_ru_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_public_relations.yaml
new file mode 100644
index 000000000..3dfd71cc3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_ru_social_sciences_tasks
+task: global_mmlu_full_ru_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_security_studies.yaml
new file mode 100644
index 000000000..bd08ea345
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_ru_social_sciences_tasks
+task: global_mmlu_full_ru_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_sociology.yaml
new file mode 100644
index 000000000..ef616ee16
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_ru_social_sciences_tasks
+task: global_mmlu_full_ru_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_us_foreign_policy.yaml
new file mode 100644
index 000000000..c8244e656
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_ru_social_sciences_tasks
+task: global_mmlu_full_ru_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_virology.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_virology.yaml
new file mode 100644
index 000000000..2f4df8105
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_ru_other_tasks
+task: global_mmlu_full_ru_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_world_religions.yaml
new file mode 100644
index 000000000..06f71986d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _ru_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_ru_humanities_tasks
+task: global_mmlu_full_ru_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/ru/utils.py b/lmms_eval/tasks/global_mmlu/full/ru/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/ru/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si.yaml b/lmms_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si.yaml
new file mode 100644
index 000000000..6503e2678
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_si
+task:
+  - global_mmlu_full_si_stem
+  - global_mmlu_full_si_other
+  - global_mmlu_full_si_social_sciences
+  - global_mmlu_full_si_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_humanities.yaml
new file mode 100644
index 000000000..b97994d04
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_si_humanities
+task:
+  - global_mmlu_full_si_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_other.yaml b/lmms_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_other.yaml
new file mode 100644
index 000000000..e7600ca45
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_si_other
+task:
+  - global_mmlu_full_si_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_social_sciences.yaml
new file mode 100644
index 000000000..4e2351a28
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_si_social_sciences
+task:
+  - global_mmlu_full_si_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_stem.yaml b/lmms_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_stem.yaml
new file mode 100644
index 000000000..8878bf808
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_si_stem
+task:
+  - global_mmlu_full_si_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/si/_si_template_yaml b/lmms_eval/tasks/global_mmlu/full/si/_si_template_yaml
new file mode 100644
index 000000000..5c775b201
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/_si_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: si
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_abstract_algebra.yaml
new file mode 100644
index 000000000..b81c58033
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_si_stem_tasks
+task: global_mmlu_full_si_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_anatomy.yaml
new file mode 100644
index 000000000..323152456
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_si_stem_tasks
+task: global_mmlu_full_si_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_astronomy.yaml
new file mode 100644
index 000000000..c7ab9539e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_si_stem_tasks
+task: global_mmlu_full_si_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_business_ethics.yaml
new file mode 100644
index 000000000..8281fc42f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_si_other_tasks
+task: global_mmlu_full_si_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_clinical_knowledge.yaml
new file mode 100644
index 000000000..2a7f5cf5b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_si_other_tasks
+task: global_mmlu_full_si_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_biology.yaml
new file mode 100644
index 000000000..e54148da1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_si_stem_tasks
+task: global_mmlu_full_si_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_chemistry.yaml
new file mode 100644
index 000000000..b797ac603
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_si_stem_tasks
+task: global_mmlu_full_si_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_computer_science.yaml
new file mode 100644
index 000000000..ba69de35d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_si_stem_tasks
+task: global_mmlu_full_si_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_mathematics.yaml
new file mode 100644
index 000000000..65ed94249
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_si_stem_tasks
+task: global_mmlu_full_si_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_medicine.yaml
new file mode 100644
index 000000000..1418aa0db
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_si_other_tasks
+task: global_mmlu_full_si_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_physics.yaml
new file mode 100644
index 000000000..cb32cd4fd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_si_stem_tasks
+task: global_mmlu_full_si_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_computer_security.yaml
new file mode 100644
index 000000000..ce5ab9b7f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_si_stem_tasks
+task: global_mmlu_full_si_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_conceptual_physics.yaml
new file mode 100644
index 000000000..c2ab5718f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_si_stem_tasks
+task: global_mmlu_full_si_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_econometrics.yaml
new file mode 100644
index 000000000..5e764903b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_si_social_sciences_tasks
+task: global_mmlu_full_si_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_electrical_engineering.yaml
new file mode 100644
index 000000000..99679bb0a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_si_stem_tasks
+task: global_mmlu_full_si_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_elementary_mathematics.yaml
new file mode 100644
index 000000000..553bc9bb7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_si_stem_tasks
+task: global_mmlu_full_si_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_formal_logic.yaml
new file mode 100644
index 000000000..112814b6c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_si_humanities_tasks
+task: global_mmlu_full_si_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_global_facts.yaml
new file mode 100644
index 000000000..008b55377
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_si_other_tasks
+task: global_mmlu_full_si_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_biology.yaml
new file mode 100644
index 000000000..fecd995a0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_si_stem_tasks
+task: global_mmlu_full_si_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_chemistry.yaml
new file mode 100644
index 000000000..3d3018b60
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_si_stem_tasks
+task: global_mmlu_full_si_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_computer_science.yaml
new file mode 100644
index 000000000..e80a1f2c4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_si_stem_tasks
+task: global_mmlu_full_si_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_european_history.yaml
new file mode 100644
index 000000000..10e157384
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_si_humanities_tasks
+task: global_mmlu_full_si_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_geography.yaml
new file mode 100644
index 000000000..12d90b97f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_si_social_sciences_tasks
+task: global_mmlu_full_si_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..d285c2c6a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_si_social_sciences_tasks
+task: global_mmlu_full_si_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..1c85f2dff
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_si_social_sciences_tasks
+task: global_mmlu_full_si_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_mathematics.yaml
new file mode 100644
index 000000000..b292fa50a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_si_stem_tasks
+task: global_mmlu_full_si_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_microeconomics.yaml
new file mode 100644
index 000000000..ada74f5f2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_si_social_sciences_tasks
+task: global_mmlu_full_si_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_physics.yaml
new file mode 100644
index 000000000..84bbda289
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_si_stem_tasks
+task: global_mmlu_full_si_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_psychology.yaml
new file mode 100644
index 000000000..7c378798c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_si_social_sciences_tasks
+task: global_mmlu_full_si_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_statistics.yaml
new file mode 100644
index 000000000..13758f22f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_si_stem_tasks
+task: global_mmlu_full_si_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_us_history.yaml
new file mode 100644
index 000000000..0fe85e14a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_si_humanities_tasks
+task: global_mmlu_full_si_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_world_history.yaml
new file mode 100644
index 000000000..8afaa392b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_si_humanities_tasks
+task: global_mmlu_full_si_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_human_aging.yaml
new file mode 100644
index 000000000..2cf69a689
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_si_other_tasks
+task: global_mmlu_full_si_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_human_sexuality.yaml
new file mode 100644
index 000000000..418927d5b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_si_social_sciences_tasks
+task: global_mmlu_full_si_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_international_law.yaml
new file mode 100644
index 000000000..de0a611da
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_si_humanities_tasks
+task: global_mmlu_full_si_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_jurisprudence.yaml
new file mode 100644
index 000000000..102121736
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_si_humanities_tasks
+task: global_mmlu_full_si_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_logical_fallacies.yaml
new file mode 100644
index 000000000..d31372ada
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_si_humanities_tasks
+task: global_mmlu_full_si_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_machine_learning.yaml
new file mode 100644
index 000000000..0e3d0e7c8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_si_stem_tasks
+task: global_mmlu_full_si_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_management.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_management.yaml
new file mode 100644
index 000000000..f4e29c9ae
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_si_other_tasks
+task: global_mmlu_full_si_management
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_marketing.yaml
new file mode 100644
index 000000000..8dff414a1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_si_other_tasks
+task: global_mmlu_full_si_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_medical_genetics.yaml
new file mode 100644
index 000000000..6160f02b4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_si_other_tasks
+task: global_mmlu_full_si_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_miscellaneous.yaml
new file mode 100644
index 000000000..de1db6c98
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_si_other_tasks
+task: global_mmlu_full_si_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_moral_disputes.yaml
new file mode 100644
index 000000000..d48cf75c9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_si_humanities_tasks
+task: global_mmlu_full_si_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_moral_scenarios.yaml
new file mode 100644
index 000000000..5d08b811f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_si_humanities_tasks
+task: global_mmlu_full_si_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_nutrition.yaml
new file mode 100644
index 000000000..3163db497
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_si_other_tasks
+task: global_mmlu_full_si_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_philosophy.yaml
new file mode 100644
index 000000000..f809bdddf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_si_humanities_tasks
+task: global_mmlu_full_si_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_prehistory.yaml
new file mode 100644
index 000000000..964e6ab7b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_si_humanities_tasks
+task: global_mmlu_full_si_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_accounting.yaml
new file mode 100644
index 000000000..c04e0bbc3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_si_other_tasks
+task: global_mmlu_full_si_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_law.yaml
new file mode 100644
index 000000000..6542f14e3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_si_humanities_tasks
+task: global_mmlu_full_si_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_medicine.yaml
new file mode 100644
index 000000000..384489798
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_si_other_tasks
+task: global_mmlu_full_si_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_psychology.yaml
new file mode 100644
index 000000000..80f36885b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_si_social_sciences_tasks
+task: global_mmlu_full_si_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_public_relations.yaml
new file mode 100644
index 000000000..2ac5169ed
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_si_social_sciences_tasks
+task: global_mmlu_full_si_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_security_studies.yaml
new file mode 100644
index 000000000..21423506a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_si_social_sciences_tasks
+task: global_mmlu_full_si_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_sociology.yaml
new file mode 100644
index 000000000..c86ee0a34
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_si_social_sciences_tasks
+task: global_mmlu_full_si_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_us_foreign_policy.yaml
new file mode 100644
index 000000000..28c238e65
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_si_social_sciences_tasks
+task: global_mmlu_full_si_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_virology.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_virology.yaml
new file mode 100644
index 000000000..a19354607
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_si_other_tasks
+task: global_mmlu_full_si_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_world_religions.yaml
new file mode 100644
index 000000000..424c23c28
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _si_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_si_humanities_tasks
+task: global_mmlu_full_si_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/si/utils.py b/lmms_eval/tasks/global_mmlu/full/si/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/si/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn.yaml b/lmms_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn.yaml
new file mode 100644
index 000000000..fd084b328
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_sn
+task:
+  - global_mmlu_full_sn_stem
+  - global_mmlu_full_sn_other
+  - global_mmlu_full_sn_social_sciences
+  - global_mmlu_full_sn_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_humanities.yaml
new file mode 100644
index 000000000..696908620
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_sn_humanities
+task:
+  - global_mmlu_full_sn_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_other.yaml b/lmms_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_other.yaml
new file mode 100644
index 000000000..18e750b62
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_sn_other
+task:
+  - global_mmlu_full_sn_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_social_sciences.yaml
new file mode 100644
index 000000000..a8e762154
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_sn_social_sciences
+task:
+  - global_mmlu_full_sn_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_stem.yaml b/lmms_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_stem.yaml
new file mode 100644
index 000000000..b3136233b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_sn_stem
+task:
+  - global_mmlu_full_sn_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/_sn_template_yaml b/lmms_eval/tasks/global_mmlu/full/sn/_sn_template_yaml
new file mode 100644
index 000000000..30d50ba0c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/_sn_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: sn
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_abstract_algebra.yaml
new file mode 100644
index 000000000..c4de495e9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_sn_stem_tasks
+task: global_mmlu_full_sn_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_anatomy.yaml
new file mode 100644
index 000000000..1ef227aa5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_sn_stem_tasks
+task: global_mmlu_full_sn_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_astronomy.yaml
new file mode 100644
index 000000000..8662ab964
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_sn_stem_tasks
+task: global_mmlu_full_sn_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_business_ethics.yaml
new file mode 100644
index 000000000..6f4741c04
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_sn_other_tasks
+task: global_mmlu_full_sn_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_clinical_knowledge.yaml
new file mode 100644
index 000000000..7477170ef
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_sn_other_tasks
+task: global_mmlu_full_sn_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_biology.yaml
new file mode 100644
index 000000000..6d0ec277c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_sn_stem_tasks
+task: global_mmlu_full_sn_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_chemistry.yaml
new file mode 100644
index 000000000..9f0c4f426
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_sn_stem_tasks
+task: global_mmlu_full_sn_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_computer_science.yaml
new file mode 100644
index 000000000..c8651ee19
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_sn_stem_tasks
+task: global_mmlu_full_sn_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_mathematics.yaml
new file mode 100644
index 000000000..c1d1a98e0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_sn_stem_tasks
+task: global_mmlu_full_sn_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_medicine.yaml
new file mode 100644
index 000000000..d9ce08f30
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_sn_other_tasks
+task: global_mmlu_full_sn_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_physics.yaml
new file mode 100644
index 000000000..ae34a82aa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_sn_stem_tasks
+task: global_mmlu_full_sn_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_computer_security.yaml
new file mode 100644
index 000000000..4b41c175e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_sn_stem_tasks
+task: global_mmlu_full_sn_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_conceptual_physics.yaml
new file mode 100644
index 000000000..5aaa8a782
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_sn_stem_tasks
+task: global_mmlu_full_sn_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_econometrics.yaml
new file mode 100644
index 000000000..8606e96c2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_sn_social_sciences_tasks
+task: global_mmlu_full_sn_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_electrical_engineering.yaml
new file mode 100644
index 000000000..9c57f7038
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_sn_stem_tasks
+task: global_mmlu_full_sn_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_elementary_mathematics.yaml
new file mode 100644
index 000000000..0ed5b400c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_sn_stem_tasks
+task: global_mmlu_full_sn_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_formal_logic.yaml
new file mode 100644
index 000000000..55dafc2bf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_sn_humanities_tasks
+task: global_mmlu_full_sn_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_global_facts.yaml
new file mode 100644
index 000000000..5b8ee96f0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_sn_other_tasks
+task: global_mmlu_full_sn_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_biology.yaml
new file mode 100644
index 000000000..2597a7d75
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_sn_stem_tasks
+task: global_mmlu_full_sn_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_chemistry.yaml
new file mode 100644
index 000000000..1e6be4e61
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_sn_stem_tasks
+task: global_mmlu_full_sn_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_computer_science.yaml
new file mode 100644
index 000000000..446da912c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_sn_stem_tasks
+task: global_mmlu_full_sn_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_european_history.yaml
new file mode 100644
index 000000000..dd8cf61c4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_sn_humanities_tasks
+task: global_mmlu_full_sn_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_geography.yaml
new file mode 100644
index 000000000..2e178adfd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_sn_social_sciences_tasks
+task: global_mmlu_full_sn_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..1ac4efda6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_sn_social_sciences_tasks
+task: global_mmlu_full_sn_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..23ca0b417
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_sn_social_sciences_tasks
+task: global_mmlu_full_sn_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_mathematics.yaml
new file mode 100644
index 000000000..0bd9be195
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_sn_stem_tasks
+task: global_mmlu_full_sn_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_microeconomics.yaml
new file mode 100644
index 000000000..916e14cae
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_sn_social_sciences_tasks
+task: global_mmlu_full_sn_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_physics.yaml
new file mode 100644
index 000000000..b6a3e60c2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_sn_stem_tasks
+task: global_mmlu_full_sn_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_psychology.yaml
new file mode 100644
index 000000000..62a197c0e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_sn_social_sciences_tasks
+task: global_mmlu_full_sn_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_statistics.yaml
new file mode 100644
index 000000000..815cb60b1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_sn_stem_tasks
+task: global_mmlu_full_sn_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_us_history.yaml
new file mode 100644
index 000000000..ff9f970e0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_sn_humanities_tasks
+task: global_mmlu_full_sn_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_world_history.yaml
new file mode 100644
index 000000000..b2dedc38d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_sn_humanities_tasks
+task: global_mmlu_full_sn_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_human_aging.yaml
new file mode 100644
index 000000000..0ef139307
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_sn_other_tasks
+task: global_mmlu_full_sn_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_human_sexuality.yaml
new file mode 100644
index 000000000..a52c2ded2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_sn_social_sciences_tasks
+task: global_mmlu_full_sn_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_international_law.yaml
new file mode 100644
index 000000000..648c3dea0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_sn_humanities_tasks
+task: global_mmlu_full_sn_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_jurisprudence.yaml
new file mode 100644
index 000000000..ca63c411a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_sn_humanities_tasks
+task: global_mmlu_full_sn_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_logical_fallacies.yaml
new file mode 100644
index 000000000..d74a7f18e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_sn_humanities_tasks
+task: global_mmlu_full_sn_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_machine_learning.yaml
new file mode 100644
index 000000000..db272b3b4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_sn_stem_tasks
+task: global_mmlu_full_sn_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_management.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_management.yaml
new file mode 100644
index 000000000..db3bee4d4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_sn_other_tasks
+task: global_mmlu_full_sn_management
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_marketing.yaml
new file mode 100644
index 000000000..a700c4e7c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_sn_other_tasks
+task: global_mmlu_full_sn_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_medical_genetics.yaml
new file mode 100644
index 000000000..b826b187e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_sn_other_tasks
+task: global_mmlu_full_sn_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_miscellaneous.yaml
new file mode 100644
index 000000000..dea895aa9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_sn_other_tasks
+task: global_mmlu_full_sn_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_moral_disputes.yaml
new file mode 100644
index 000000000..b641f6b2a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_sn_humanities_tasks
+task: global_mmlu_full_sn_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_moral_scenarios.yaml
new file mode 100644
index 000000000..2951a953c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_sn_humanities_tasks
+task: global_mmlu_full_sn_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_nutrition.yaml
new file mode 100644
index 000000000..9816d8b48
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_sn_other_tasks
+task: global_mmlu_full_sn_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_philosophy.yaml
new file mode 100644
index 000000000..4ea105050
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_sn_humanities_tasks
+task: global_mmlu_full_sn_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_prehistory.yaml
new file mode 100644
index 000000000..e941437be
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_sn_humanities_tasks
+task: global_mmlu_full_sn_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_accounting.yaml
new file mode 100644
index 000000000..057a197db
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_sn_other_tasks
+task: global_mmlu_full_sn_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_law.yaml
new file mode 100644
index 000000000..72c9fac75
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_sn_humanities_tasks
+task: global_mmlu_full_sn_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_medicine.yaml
new file mode 100644
index 000000000..e727b3cd5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_sn_other_tasks
+task: global_mmlu_full_sn_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_psychology.yaml
new file mode 100644
index 000000000..341322d2c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_sn_social_sciences_tasks
+task: global_mmlu_full_sn_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_public_relations.yaml
new file mode 100644
index 000000000..5448baa47
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_sn_social_sciences_tasks
+task: global_mmlu_full_sn_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_security_studies.yaml
new file mode 100644
index 000000000..542c709af
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_sn_social_sciences_tasks
+task: global_mmlu_full_sn_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_sociology.yaml
new file mode 100644
index 000000000..f2913db53
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_sn_social_sciences_tasks
+task: global_mmlu_full_sn_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_us_foreign_policy.yaml
new file mode 100644
index 000000000..ad4768478
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_sn_social_sciences_tasks
+task: global_mmlu_full_sn_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_virology.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_virology.yaml
new file mode 100644
index 000000000..254fedb4f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_sn_other_tasks
+task: global_mmlu_full_sn_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_world_religions.yaml
new file mode 100644
index 000000000..2aef6dfdd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sn_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_sn_humanities_tasks
+task: global_mmlu_full_sn_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/sn/utils.py b/lmms_eval/tasks/global_mmlu/full/sn/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sn/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so.yaml b/lmms_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so.yaml
new file mode 100644
index 000000000..91e11b88e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_so
+task:
+  - global_mmlu_full_so_stem
+  - global_mmlu_full_so_other
+  - global_mmlu_full_so_social_sciences
+  - global_mmlu_full_so_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_humanities.yaml
new file mode 100644
index 000000000..ff78bfaba
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_so_humanities
+task:
+  - global_mmlu_full_so_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_other.yaml b/lmms_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_other.yaml
new file mode 100644
index 000000000..eec8e6613
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_so_other
+task:
+  - global_mmlu_full_so_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_social_sciences.yaml
new file mode 100644
index 000000000..9d00ea1f5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_so_social_sciences
+task:
+  - global_mmlu_full_so_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_stem.yaml b/lmms_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_stem.yaml
new file mode 100644
index 000000000..497b9b01e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_so_stem
+task:
+  - global_mmlu_full_so_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/so/_so_template_yaml b/lmms_eval/tasks/global_mmlu/full/so/_so_template_yaml
new file mode 100644
index 000000000..fb052a633
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/_so_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: so
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_abstract_algebra.yaml
new file mode 100644
index 000000000..afb5d9084
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_so_stem_tasks
+task: global_mmlu_full_so_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_anatomy.yaml
new file mode 100644
index 000000000..79f3446df
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_so_stem_tasks
+task: global_mmlu_full_so_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_astronomy.yaml
new file mode 100644
index 000000000..54a2faa09
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_so_stem_tasks
+task: global_mmlu_full_so_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_business_ethics.yaml
new file mode 100644
index 000000000..65bc598cb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_so_other_tasks
+task: global_mmlu_full_so_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_clinical_knowledge.yaml
new file mode 100644
index 000000000..224aa39b2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_so_other_tasks
+task: global_mmlu_full_so_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_biology.yaml
new file mode 100644
index 000000000..758d22c37
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_so_stem_tasks
+task: global_mmlu_full_so_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_chemistry.yaml
new file mode 100644
index 000000000..35c224308
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_so_stem_tasks
+task: global_mmlu_full_so_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_computer_science.yaml
new file mode 100644
index 000000000..86428ae82
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_so_stem_tasks
+task: global_mmlu_full_so_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_mathematics.yaml
new file mode 100644
index 000000000..f9957a234
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_so_stem_tasks
+task: global_mmlu_full_so_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_medicine.yaml
new file mode 100644
index 000000000..f51a1b123
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_so_other_tasks
+task: global_mmlu_full_so_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_physics.yaml
new file mode 100644
index 000000000..43388d6c8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_so_stem_tasks
+task: global_mmlu_full_so_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_computer_security.yaml
new file mode 100644
index 000000000..8a5563308
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_so_stem_tasks
+task: global_mmlu_full_so_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_conceptual_physics.yaml
new file mode 100644
index 000000000..97dfa147b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_so_stem_tasks
+task: global_mmlu_full_so_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_econometrics.yaml
new file mode 100644
index 000000000..9792659fa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_so_social_sciences_tasks
+task: global_mmlu_full_so_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_electrical_engineering.yaml
new file mode 100644
index 000000000..3ed44e41f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_so_stem_tasks
+task: global_mmlu_full_so_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_elementary_mathematics.yaml
new file mode 100644
index 000000000..76628481f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_so_stem_tasks
+task: global_mmlu_full_so_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_formal_logic.yaml
new file mode 100644
index 000000000..4b7645c8e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_so_humanities_tasks
+task: global_mmlu_full_so_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_global_facts.yaml
new file mode 100644
index 000000000..fa75e6664
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_so_other_tasks
+task: global_mmlu_full_so_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_biology.yaml
new file mode 100644
index 000000000..d3ad29d4c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_so_stem_tasks
+task: global_mmlu_full_so_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_chemistry.yaml
new file mode 100644
index 000000000..274af23bf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_so_stem_tasks
+task: global_mmlu_full_so_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_computer_science.yaml
new file mode 100644
index 000000000..6bce30d68
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_so_stem_tasks
+task: global_mmlu_full_so_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_european_history.yaml
new file mode 100644
index 000000000..cfc44f08d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_so_humanities_tasks
+task: global_mmlu_full_so_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_geography.yaml
new file mode 100644
index 000000000..55479c399
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_so_social_sciences_tasks
+task: global_mmlu_full_so_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..ceb5a7014
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_so_social_sciences_tasks
+task: global_mmlu_full_so_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..0c403ec51
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_so_social_sciences_tasks
+task: global_mmlu_full_so_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_mathematics.yaml
new file mode 100644
index 000000000..e8089bdf8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_so_stem_tasks
+task: global_mmlu_full_so_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_microeconomics.yaml
new file mode 100644
index 000000000..32cacffea
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_so_social_sciences_tasks
+task: global_mmlu_full_so_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_physics.yaml
new file mode 100644
index 000000000..fd2c35ac8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_so_stem_tasks
+task: global_mmlu_full_so_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_psychology.yaml
new file mode 100644
index 000000000..26f2cb3c4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_so_social_sciences_tasks
+task: global_mmlu_full_so_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_statistics.yaml
new file mode 100644
index 000000000..730075b1c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_so_stem_tasks
+task: global_mmlu_full_so_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_us_history.yaml
new file mode 100644
index 000000000..c9702a661
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_so_humanities_tasks
+task: global_mmlu_full_so_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_world_history.yaml
new file mode 100644
index 000000000..78a21d5d9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_so_humanities_tasks
+task: global_mmlu_full_so_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_human_aging.yaml
new file mode 100644
index 000000000..c95b5562a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_so_other_tasks
+task: global_mmlu_full_so_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_human_sexuality.yaml
new file mode 100644
index 000000000..632778d3b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_so_social_sciences_tasks
+task: global_mmlu_full_so_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_international_law.yaml
new file mode 100644
index 000000000..2d5ab1c53
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_so_humanities_tasks
+task: global_mmlu_full_so_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_jurisprudence.yaml
new file mode 100644
index 000000000..1372a1d17
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_so_humanities_tasks
+task: global_mmlu_full_so_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_logical_fallacies.yaml
new file mode 100644
index 000000000..19a1120eb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_so_humanities_tasks
+task: global_mmlu_full_so_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_machine_learning.yaml
new file mode 100644
index 000000000..c1e13ddad
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_so_stem_tasks
+task: global_mmlu_full_so_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_management.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_management.yaml
new file mode 100644
index 000000000..6e3252056
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_so_other_tasks
+task: global_mmlu_full_so_management
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_marketing.yaml
new file mode 100644
index 000000000..8b1c002fe
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_so_other_tasks
+task: global_mmlu_full_so_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_medical_genetics.yaml
new file mode 100644
index 000000000..c0136dc69
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_so_other_tasks
+task: global_mmlu_full_so_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_miscellaneous.yaml
new file mode 100644
index 000000000..2b8a33ba9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_so_other_tasks
+task: global_mmlu_full_so_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_moral_disputes.yaml
new file mode 100644
index 000000000..c1bd0011b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_so_humanities_tasks
+task: global_mmlu_full_so_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_moral_scenarios.yaml
new file mode 100644
index 000000000..60418a65f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_so_humanities_tasks
+task: global_mmlu_full_so_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_nutrition.yaml
new file mode 100644
index 000000000..5aa40241a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_so_other_tasks
+task: global_mmlu_full_so_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_philosophy.yaml
new file mode 100644
index 000000000..421a98016
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_so_humanities_tasks
+task: global_mmlu_full_so_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_prehistory.yaml
new file mode 100644
index 000000000..721bfbf22
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_so_humanities_tasks
+task: global_mmlu_full_so_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_accounting.yaml
new file mode 100644
index 000000000..4ca0c5c9b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_so_other_tasks
+task: global_mmlu_full_so_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_law.yaml
new file mode 100644
index 000000000..7f57b5949
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_so_humanities_tasks
+task: global_mmlu_full_so_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_medicine.yaml
new file mode 100644
index 000000000..a7d6408e2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_so_other_tasks
+task: global_mmlu_full_so_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_psychology.yaml
new file mode 100644
index 000000000..a03de5bb9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_so_social_sciences_tasks
+task: global_mmlu_full_so_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_public_relations.yaml
new file mode 100644
index 000000000..f7af81e6a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_so_social_sciences_tasks
+task: global_mmlu_full_so_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_security_studies.yaml
new file mode 100644
index 000000000..b52ee2590
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_so_social_sciences_tasks
+task: global_mmlu_full_so_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_sociology.yaml
new file mode 100644
index 000000000..7f3847e67
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_so_social_sciences_tasks
+task: global_mmlu_full_so_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_us_foreign_policy.yaml
new file mode 100644
index 000000000..a6017167a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_so_social_sciences_tasks
+task: global_mmlu_full_so_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_virology.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_virology.yaml
new file mode 100644
index 000000000..2dc85b324
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_so_other_tasks
+task: global_mmlu_full_so_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_world_religions.yaml
new file mode 100644
index 000000000..9ca99e5b4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _so_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_so_humanities_tasks
+task: global_mmlu_full_so_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/so/utils.py b/lmms_eval/tasks/global_mmlu/full/so/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/so/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr.yaml b/lmms_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr.yaml
new file mode 100644
index 000000000..1e0c55a5d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_sr
+task:
+  - global_mmlu_full_sr_stem
+  - global_mmlu_full_sr_other
+  - global_mmlu_full_sr_social_sciences
+  - global_mmlu_full_sr_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_humanities.yaml
new file mode 100644
index 000000000..080bc545c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_sr_humanities
+task:
+  - global_mmlu_full_sr_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_other.yaml b/lmms_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_other.yaml
new file mode 100644
index 000000000..9f0735eb4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_sr_other
+task:
+  - global_mmlu_full_sr_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_social_sciences.yaml
new file mode 100644
index 000000000..bdc29d1f6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_sr_social_sciences
+task:
+  - global_mmlu_full_sr_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_stem.yaml b/lmms_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_stem.yaml
new file mode 100644
index 000000000..7c4aa6369
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_sr_stem
+task:
+  - global_mmlu_full_sr_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/_sr_template_yaml b/lmms_eval/tasks/global_mmlu/full/sr/_sr_template_yaml
new file mode 100644
index 000000000..6af61b3b2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/_sr_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: sr
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_abstract_algebra.yaml
new file mode 100644
index 000000000..b3275870e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_sr_stem_tasks
+task: global_mmlu_full_sr_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_anatomy.yaml
new file mode 100644
index 000000000..5689af736
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_sr_stem_tasks
+task: global_mmlu_full_sr_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_astronomy.yaml
new file mode 100644
index 000000000..3d23a4383
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_sr_stem_tasks
+task: global_mmlu_full_sr_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_business_ethics.yaml
new file mode 100644
index 000000000..e89f5e61b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_sr_other_tasks
+task: global_mmlu_full_sr_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_clinical_knowledge.yaml
new file mode 100644
index 000000000..b5611c15c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_sr_other_tasks
+task: global_mmlu_full_sr_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_biology.yaml
new file mode 100644
index 000000000..9e28c3034
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_sr_stem_tasks
+task: global_mmlu_full_sr_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_chemistry.yaml
new file mode 100644
index 000000000..1eac952c9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_sr_stem_tasks
+task: global_mmlu_full_sr_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_computer_science.yaml
new file mode 100644
index 000000000..e1146aa11
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_sr_stem_tasks
+task: global_mmlu_full_sr_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_mathematics.yaml
new file mode 100644
index 000000000..bcfda2ba3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_sr_stem_tasks
+task: global_mmlu_full_sr_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_medicine.yaml
new file mode 100644
index 000000000..3beb5b26c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_sr_other_tasks
+task: global_mmlu_full_sr_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_physics.yaml
new file mode 100644
index 000000000..f959a02fc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_sr_stem_tasks
+task: global_mmlu_full_sr_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_computer_security.yaml
new file mode 100644
index 000000000..7e8761e0d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_sr_stem_tasks
+task: global_mmlu_full_sr_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_conceptual_physics.yaml
new file mode 100644
index 000000000..9325f6de7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_sr_stem_tasks
+task: global_mmlu_full_sr_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_econometrics.yaml
new file mode 100644
index 000000000..cc4a5bcc3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_sr_social_sciences_tasks
+task: global_mmlu_full_sr_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_electrical_engineering.yaml
new file mode 100644
index 000000000..d3a5a78b5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_sr_stem_tasks
+task: global_mmlu_full_sr_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_elementary_mathematics.yaml
new file mode 100644
index 000000000..50f60166c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_sr_stem_tasks
+task: global_mmlu_full_sr_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_formal_logic.yaml
new file mode 100644
index 000000000..8bdd854f1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_sr_humanities_tasks
+task: global_mmlu_full_sr_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_global_facts.yaml
new file mode 100644
index 000000000..88862d212
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_sr_other_tasks
+task: global_mmlu_full_sr_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_biology.yaml
new file mode 100644
index 000000000..8f2b2952b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_sr_stem_tasks
+task: global_mmlu_full_sr_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_chemistry.yaml
new file mode 100644
index 000000000..6b89deb12
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_sr_stem_tasks
+task: global_mmlu_full_sr_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_computer_science.yaml
new file mode 100644
index 000000000..55fd7e8e6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_sr_stem_tasks
+task: global_mmlu_full_sr_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_european_history.yaml
new file mode 100644
index 000000000..946acf0ee
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_sr_humanities_tasks
+task: global_mmlu_full_sr_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_geography.yaml
new file mode 100644
index 000000000..070589715
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_sr_social_sciences_tasks
+task: global_mmlu_full_sr_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..a9721c9ba
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_sr_social_sciences_tasks
+task: global_mmlu_full_sr_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..fedea95a6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_sr_social_sciences_tasks
+task: global_mmlu_full_sr_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_mathematics.yaml
new file mode 100644
index 000000000..dca9e1402
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_sr_stem_tasks
+task: global_mmlu_full_sr_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_microeconomics.yaml
new file mode 100644
index 000000000..b01276f65
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_sr_social_sciences_tasks
+task: global_mmlu_full_sr_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_physics.yaml
new file mode 100644
index 000000000..f549f8ac2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_sr_stem_tasks
+task: global_mmlu_full_sr_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_psychology.yaml
new file mode 100644
index 000000000..c6b31eeed
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_sr_social_sciences_tasks
+task: global_mmlu_full_sr_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_statistics.yaml
new file mode 100644
index 000000000..12d0f0e5c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_sr_stem_tasks
+task: global_mmlu_full_sr_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_us_history.yaml
new file mode 100644
index 000000000..98c401001
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_sr_humanities_tasks
+task: global_mmlu_full_sr_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_world_history.yaml
new file mode 100644
index 000000000..76e6b45c8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_sr_humanities_tasks
+task: global_mmlu_full_sr_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_human_aging.yaml
new file mode 100644
index 000000000..b0ff1d95a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_sr_other_tasks
+task: global_mmlu_full_sr_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_human_sexuality.yaml
new file mode 100644
index 000000000..73a30099c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_sr_social_sciences_tasks
+task: global_mmlu_full_sr_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_international_law.yaml
new file mode 100644
index 000000000..0aea0826f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_sr_humanities_tasks
+task: global_mmlu_full_sr_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_jurisprudence.yaml
new file mode 100644
index 000000000..debe604fe
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_sr_humanities_tasks
+task: global_mmlu_full_sr_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_logical_fallacies.yaml
new file mode 100644
index 000000000..407417f3e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_sr_humanities_tasks
+task: global_mmlu_full_sr_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_machine_learning.yaml
new file mode 100644
index 000000000..513a7f877
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_sr_stem_tasks
+task: global_mmlu_full_sr_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_management.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_management.yaml
new file mode 100644
index 000000000..fca9de044
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_sr_other_tasks
+task: global_mmlu_full_sr_management
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_marketing.yaml
new file mode 100644
index 000000000..8267563e6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_sr_other_tasks
+task: global_mmlu_full_sr_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_medical_genetics.yaml
new file mode 100644
index 000000000..4ba860f2d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_sr_other_tasks
+task: global_mmlu_full_sr_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_miscellaneous.yaml
new file mode 100644
index 000000000..ecdbcea91
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_sr_other_tasks
+task: global_mmlu_full_sr_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_moral_disputes.yaml
new file mode 100644
index 000000000..54bf34916
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_sr_humanities_tasks
+task: global_mmlu_full_sr_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_moral_scenarios.yaml
new file mode 100644
index 000000000..2eab8d4a0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_sr_humanities_tasks
+task: global_mmlu_full_sr_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_nutrition.yaml
new file mode 100644
index 000000000..83e1b84c4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_sr_other_tasks
+task: global_mmlu_full_sr_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_philosophy.yaml
new file mode 100644
index 000000000..654ee86b9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_sr_humanities_tasks
+task: global_mmlu_full_sr_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_prehistory.yaml
new file mode 100644
index 000000000..3a2f944b7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_sr_humanities_tasks
+task: global_mmlu_full_sr_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_accounting.yaml
new file mode 100644
index 000000000..648ae0cb0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_sr_other_tasks
+task: global_mmlu_full_sr_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_law.yaml
new file mode 100644
index 000000000..0ee8a831b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_sr_humanities_tasks
+task: global_mmlu_full_sr_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_medicine.yaml
new file mode 100644
index 000000000..3b1421154
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_sr_other_tasks
+task: global_mmlu_full_sr_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_psychology.yaml
new file mode 100644
index 000000000..19e2dc544
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_sr_social_sciences_tasks
+task: global_mmlu_full_sr_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_public_relations.yaml
new file mode 100644
index 000000000..043024c0d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_sr_social_sciences_tasks
+task: global_mmlu_full_sr_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_security_studies.yaml
new file mode 100644
index 000000000..24720925a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_sr_social_sciences_tasks
+task: global_mmlu_full_sr_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_sociology.yaml
new file mode 100644
index 000000000..fc93c5e07
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_sr_social_sciences_tasks
+task: global_mmlu_full_sr_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_us_foreign_policy.yaml
new file mode 100644
index 000000000..1b338dd6b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_sr_social_sciences_tasks
+task: global_mmlu_full_sr_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_virology.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_virology.yaml
new file mode 100644
index 000000000..b07588ad1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_sr_other_tasks
+task: global_mmlu_full_sr_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_world_religions.yaml
new file mode 100644
index 000000000..3f78403ed
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sr_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_sr_humanities_tasks
+task: global_mmlu_full_sr_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/sr/utils.py b/lmms_eval/tasks/global_mmlu/full/sr/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sr/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv.yaml b/lmms_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv.yaml
new file mode 100644
index 000000000..078de8b63
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_sv
+task:
+  - global_mmlu_full_sv_stem
+  - global_mmlu_full_sv_other
+  - global_mmlu_full_sv_social_sciences
+  - global_mmlu_full_sv_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_humanities.yaml
new file mode 100644
index 000000000..f8b4628f7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_sv_humanities
+task:
+  - global_mmlu_full_sv_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_other.yaml b/lmms_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_other.yaml
new file mode 100644
index 000000000..1b29ca130
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_sv_other
+task:
+  - global_mmlu_full_sv_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_social_sciences.yaml
new file mode 100644
index 000000000..7c4a813e8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_sv_social_sciences
+task:
+  - global_mmlu_full_sv_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_stem.yaml b/lmms_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_stem.yaml
new file mode 100644
index 000000000..a6fd88f1e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_sv_stem
+task:
+  - global_mmlu_full_sv_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/_sv_template_yaml b/lmms_eval/tasks/global_mmlu/full/sv/_sv_template_yaml
new file mode 100644
index 000000000..1b9fdea98
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/_sv_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: sv
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_abstract_algebra.yaml
new file mode 100644
index 000000000..8329302f5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_sv_stem_tasks
+task: global_mmlu_full_sv_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_anatomy.yaml
new file mode 100644
index 000000000..ac9fa560e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_sv_stem_tasks
+task: global_mmlu_full_sv_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_astronomy.yaml
new file mode 100644
index 000000000..096e0e8f0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_sv_stem_tasks
+task: global_mmlu_full_sv_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_business_ethics.yaml
new file mode 100644
index 000000000..ced0b051f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_sv_other_tasks
+task: global_mmlu_full_sv_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_clinical_knowledge.yaml
new file mode 100644
index 000000000..a88871b4e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_sv_other_tasks
+task: global_mmlu_full_sv_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_biology.yaml
new file mode 100644
index 000000000..c2462c179
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_sv_stem_tasks
+task: global_mmlu_full_sv_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_chemistry.yaml
new file mode 100644
index 000000000..3ae3fecd4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_sv_stem_tasks
+task: global_mmlu_full_sv_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_computer_science.yaml
new file mode 100644
index 000000000..a3f00b249
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_sv_stem_tasks
+task: global_mmlu_full_sv_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_mathematics.yaml
new file mode 100644
index 000000000..71f613d4a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_sv_stem_tasks
+task: global_mmlu_full_sv_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_medicine.yaml
new file mode 100644
index 000000000..46f4c6eaa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_sv_other_tasks
+task: global_mmlu_full_sv_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_physics.yaml
new file mode 100644
index 000000000..06906bfd3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_sv_stem_tasks
+task: global_mmlu_full_sv_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_computer_security.yaml
new file mode 100644
index 000000000..1013ef300
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_sv_stem_tasks
+task: global_mmlu_full_sv_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_conceptual_physics.yaml
new file mode 100644
index 000000000..a6a752f0a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_sv_stem_tasks
+task: global_mmlu_full_sv_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_econometrics.yaml
new file mode 100644
index 000000000..547365f63
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_sv_social_sciences_tasks
+task: global_mmlu_full_sv_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_electrical_engineering.yaml
new file mode 100644
index 000000000..74086a15a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_sv_stem_tasks
+task: global_mmlu_full_sv_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_elementary_mathematics.yaml
new file mode 100644
index 000000000..8d1f4847c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_sv_stem_tasks
+task: global_mmlu_full_sv_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_formal_logic.yaml
new file mode 100644
index 000000000..b78b5846b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_sv_humanities_tasks
+task: global_mmlu_full_sv_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_global_facts.yaml
new file mode 100644
index 000000000..dd2056294
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_sv_other_tasks
+task: global_mmlu_full_sv_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_biology.yaml
new file mode 100644
index 000000000..fc6ebf2fd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_sv_stem_tasks
+task: global_mmlu_full_sv_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_chemistry.yaml
new file mode 100644
index 000000000..03773a83b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_sv_stem_tasks
+task: global_mmlu_full_sv_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_computer_science.yaml
new file mode 100644
index 000000000..e3db653a9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_sv_stem_tasks
+task: global_mmlu_full_sv_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_european_history.yaml
new file mode 100644
index 000000000..4a087557a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_sv_humanities_tasks
+task: global_mmlu_full_sv_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_geography.yaml
new file mode 100644
index 000000000..638553848
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_sv_social_sciences_tasks
+task: global_mmlu_full_sv_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..7e62f26f0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_sv_social_sciences_tasks
+task: global_mmlu_full_sv_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..b686a26e6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_sv_social_sciences_tasks
+task: global_mmlu_full_sv_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_mathematics.yaml
new file mode 100644
index 000000000..177165383
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_sv_stem_tasks
+task: global_mmlu_full_sv_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_microeconomics.yaml
new file mode 100644
index 000000000..e9817c173
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_sv_social_sciences_tasks
+task: global_mmlu_full_sv_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_physics.yaml
new file mode 100644
index 000000000..61359149c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_sv_stem_tasks
+task: global_mmlu_full_sv_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_psychology.yaml
new file mode 100644
index 000000000..ce3aa9e20
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_sv_social_sciences_tasks
+task: global_mmlu_full_sv_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_statistics.yaml
new file mode 100644
index 000000000..6f705f8e1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_sv_stem_tasks
+task: global_mmlu_full_sv_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_us_history.yaml
new file mode 100644
index 000000000..765cdf603
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_sv_humanities_tasks
+task: global_mmlu_full_sv_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_world_history.yaml
new file mode 100644
index 000000000..de7b30b4e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_sv_humanities_tasks
+task: global_mmlu_full_sv_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_human_aging.yaml
new file mode 100644
index 000000000..20969051b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_sv_other_tasks
+task: global_mmlu_full_sv_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_human_sexuality.yaml
new file mode 100644
index 000000000..a8bd5fab3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_sv_social_sciences_tasks
+task: global_mmlu_full_sv_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_international_law.yaml
new file mode 100644
index 000000000..7e5ddb57e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_sv_humanities_tasks
+task: global_mmlu_full_sv_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_jurisprudence.yaml
new file mode 100644
index 000000000..ff161d5f8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_sv_humanities_tasks
+task: global_mmlu_full_sv_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_logical_fallacies.yaml
new file mode 100644
index 000000000..f1602c908
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_sv_humanities_tasks
+task: global_mmlu_full_sv_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_machine_learning.yaml
new file mode 100644
index 000000000..6f011063f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_sv_stem_tasks
+task: global_mmlu_full_sv_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_management.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_management.yaml
new file mode 100644
index 000000000..7ff7b8736
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_sv_other_tasks
+task: global_mmlu_full_sv_management
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_marketing.yaml
new file mode 100644
index 000000000..c0e669f71
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_sv_other_tasks
+task: global_mmlu_full_sv_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_medical_genetics.yaml
new file mode 100644
index 000000000..83e524457
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_sv_other_tasks
+task: global_mmlu_full_sv_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_miscellaneous.yaml
new file mode 100644
index 000000000..f17987920
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_sv_other_tasks
+task: global_mmlu_full_sv_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_moral_disputes.yaml
new file mode 100644
index 000000000..1f03ac090
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_sv_humanities_tasks
+task: global_mmlu_full_sv_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_moral_scenarios.yaml
new file mode 100644
index 000000000..fe7f58d23
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_sv_humanities_tasks
+task: global_mmlu_full_sv_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_nutrition.yaml
new file mode 100644
index 000000000..79207a877
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_sv_other_tasks
+task: global_mmlu_full_sv_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_philosophy.yaml
new file mode 100644
index 000000000..ae533079d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_sv_humanities_tasks
+task: global_mmlu_full_sv_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_prehistory.yaml
new file mode 100644
index 000000000..1c602c4fe
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_sv_humanities_tasks
+task: global_mmlu_full_sv_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_accounting.yaml
new file mode 100644
index 000000000..ebdef8a89
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_sv_other_tasks
+task: global_mmlu_full_sv_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_law.yaml
new file mode 100644
index 000000000..3645c38a3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_sv_humanities_tasks
+task: global_mmlu_full_sv_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_medicine.yaml
new file mode 100644
index 000000000..d40f577df
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_sv_other_tasks
+task: global_mmlu_full_sv_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_psychology.yaml
new file mode 100644
index 000000000..edf831060
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_sv_social_sciences_tasks
+task: global_mmlu_full_sv_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_public_relations.yaml
new file mode 100644
index 000000000..f897662c2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_sv_social_sciences_tasks
+task: global_mmlu_full_sv_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_security_studies.yaml
new file mode 100644
index 000000000..9ad4fb5cc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_sv_social_sciences_tasks
+task: global_mmlu_full_sv_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_sociology.yaml
new file mode 100644
index 000000000..4b8696060
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_sv_social_sciences_tasks
+task: global_mmlu_full_sv_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_us_foreign_policy.yaml
new file mode 100644
index 000000000..522778def
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_sv_social_sciences_tasks
+task: global_mmlu_full_sv_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_virology.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_virology.yaml
new file mode 100644
index 000000000..8b3cbc8dd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_sv_other_tasks
+task: global_mmlu_full_sv_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_world_religions.yaml
new file mode 100644
index 000000000..1d7df52b0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sv_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_sv_humanities_tasks
+task: global_mmlu_full_sv_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/sv/utils.py b/lmms_eval/tasks/global_mmlu/full/sv/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sv/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/_global_mmlu_full_sw.yaml b/lmms_eval/tasks/global_mmlu/full/sw/_global_mmlu_full_sw.yaml
new file mode 100644
index 000000000..fd87d0682
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/_global_mmlu_full_sw.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_sw
+task:
+  - global_mmlu_full_sw_stem
+  - global_mmlu_full_sw_other
+  - global_mmlu_full_sw_social_sciences
+  - global_mmlu_full_sw_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_humanities.yaml
new file mode 100644
index 000000000..02168dff1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_sw_humanities
+task:
+  - global_mmlu_full_sw_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_other.yaml b/lmms_eval/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_other.yaml
new file mode 100644
index 000000000..9fa28a162
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_sw_other
+task:
+  - global_mmlu_full_sw_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_social_sciences.yaml
new file mode 100644
index 000000000..ad3184423
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_sw_social_sciences
+task:
+  - global_mmlu_full_sw_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_stem.yaml b/lmms_eval/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_stem.yaml
new file mode 100644
index 000000000..6f23cae88
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_sw_stem
+task:
+  - global_mmlu_full_sw_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/_sw_template_yaml b/lmms_eval/tasks/global_mmlu/full/sw/_sw_template_yaml
new file mode 100644
index 000000000..58cf53220
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/_sw_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: sw
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_abstract_algebra.yaml
new file mode 100644
index 000000000..187229fb5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_sw_stem_tasks
+task: global_mmlu_full_sw_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_anatomy.yaml
new file mode 100644
index 000000000..3d0d4c5c5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_sw_stem_tasks
+task: global_mmlu_full_sw_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_astronomy.yaml
new file mode 100644
index 000000000..0639b3905
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_sw_stem_tasks
+task: global_mmlu_full_sw_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_business_ethics.yaml
new file mode 100644
index 000000000..a729c9da0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_sw_other_tasks
+task: global_mmlu_full_sw_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_clinical_knowledge.yaml
new file mode 100644
index 000000000..c6b83623f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_sw_other_tasks
+task: global_mmlu_full_sw_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_biology.yaml
new file mode 100644
index 000000000..1856b9343
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_sw_stem_tasks
+task: global_mmlu_full_sw_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_chemistry.yaml
new file mode 100644
index 000000000..5ad547ff8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_sw_stem_tasks
+task: global_mmlu_full_sw_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_computer_science.yaml
new file mode 100644
index 000000000..ff8d87419
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_sw_stem_tasks
+task: global_mmlu_full_sw_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_mathematics.yaml
new file mode 100644
index 000000000..02f53a4a1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_sw_stem_tasks
+task: global_mmlu_full_sw_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_medicine.yaml
new file mode 100644
index 000000000..b9f4cc6cb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_sw_other_tasks
+task: global_mmlu_full_sw_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_physics.yaml
new file mode 100644
index 000000000..bcca5b3f9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_sw_stem_tasks
+task: global_mmlu_full_sw_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_computer_security.yaml
new file mode 100644
index 000000000..434d2faa5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_sw_stem_tasks
+task: global_mmlu_full_sw_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_conceptual_physics.yaml
new file mode 100644
index 000000000..2c1c9d41e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_sw_stem_tasks
+task: global_mmlu_full_sw_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_econometrics.yaml
new file mode 100644
index 000000000..2a907de64
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_sw_social_sciences_tasks
+task: global_mmlu_full_sw_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_electrical_engineering.yaml
new file mode 100644
index 000000000..1ae86a7c2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_sw_stem_tasks
+task: global_mmlu_full_sw_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_elementary_mathematics.yaml
new file mode 100644
index 000000000..05871f250
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_sw_stem_tasks
+task: global_mmlu_full_sw_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_formal_logic.yaml
new file mode 100644
index 000000000..8d0de4076
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_sw_humanities_tasks
+task: global_mmlu_full_sw_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_global_facts.yaml
new file mode 100644
index 000000000..29bec055e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_sw_other_tasks
+task: global_mmlu_full_sw_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_biology.yaml
new file mode 100644
index 000000000..2e49866a9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_sw_stem_tasks
+task: global_mmlu_full_sw_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_chemistry.yaml
new file mode 100644
index 000000000..a7adbd978
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_sw_stem_tasks
+task: global_mmlu_full_sw_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_computer_science.yaml
new file mode 100644
index 000000000..2e65ab5ab
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_sw_stem_tasks
+task: global_mmlu_full_sw_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_european_history.yaml
new file mode 100644
index 000000000..7352ad72e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_sw_humanities_tasks
+task: global_mmlu_full_sw_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_geography.yaml
new file mode 100644
index 000000000..797932bae
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_sw_social_sciences_tasks
+task: global_mmlu_full_sw_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..602d71ffa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_sw_social_sciences_tasks
+task: global_mmlu_full_sw_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..a91dd829b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_sw_social_sciences_tasks
+task: global_mmlu_full_sw_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_mathematics.yaml
new file mode 100644
index 000000000..c19b28da6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_sw_stem_tasks
+task: global_mmlu_full_sw_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_microeconomics.yaml
new file mode 100644
index 000000000..7a9c63bd0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_sw_social_sciences_tasks
+task: global_mmlu_full_sw_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_physics.yaml
new file mode 100644
index 000000000..239eac65f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_sw_stem_tasks
+task: global_mmlu_full_sw_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_psychology.yaml
new file mode 100644
index 000000000..b4f19d84b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_sw_social_sciences_tasks
+task: global_mmlu_full_sw_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_statistics.yaml
new file mode 100644
index 000000000..5725af637
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_sw_stem_tasks
+task: global_mmlu_full_sw_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_us_history.yaml
new file mode 100644
index 000000000..1d0803407
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_sw_humanities_tasks
+task: global_mmlu_full_sw_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_world_history.yaml
new file mode 100644
index 000000000..cfe5a9e7d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_sw_humanities_tasks
+task: global_mmlu_full_sw_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_human_aging.yaml
new file mode 100644
index 000000000..ba20e932f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_sw_other_tasks
+task: global_mmlu_full_sw_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_human_sexuality.yaml
new file mode 100644
index 000000000..4609bea09
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_sw_social_sciences_tasks
+task: global_mmlu_full_sw_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_international_law.yaml
new file mode 100644
index 000000000..bbf616b1d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_sw_humanities_tasks
+task: global_mmlu_full_sw_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_jurisprudence.yaml
new file mode 100644
index 000000000..6781f2d52
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_sw_humanities_tasks
+task: global_mmlu_full_sw_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_logical_fallacies.yaml
new file mode 100644
index 000000000..1f862917f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_sw_humanities_tasks
+task: global_mmlu_full_sw_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_machine_learning.yaml
new file mode 100644
index 000000000..9eb51cfbc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_sw_stem_tasks
+task: global_mmlu_full_sw_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_management.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_management.yaml
new file mode 100644
index 000000000..5b0e9e673
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_sw_other_tasks
+task: global_mmlu_full_sw_management
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_marketing.yaml
new file mode 100644
index 000000000..fb65e87eb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_sw_other_tasks
+task: global_mmlu_full_sw_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_medical_genetics.yaml
new file mode 100644
index 000000000..10d4db0d8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_sw_other_tasks
+task: global_mmlu_full_sw_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_miscellaneous.yaml
new file mode 100644
index 000000000..b337d0ab8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_sw_other_tasks
+task: global_mmlu_full_sw_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_moral_disputes.yaml
new file mode 100644
index 000000000..f44bfa0d9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_sw_humanities_tasks
+task: global_mmlu_full_sw_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_moral_scenarios.yaml
new file mode 100644
index 000000000..eabd5a916
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_sw_humanities_tasks
+task: global_mmlu_full_sw_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_nutrition.yaml
new file mode 100644
index 000000000..41c644580
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_sw_other_tasks
+task: global_mmlu_full_sw_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_philosophy.yaml
new file mode 100644
index 000000000..96edac994
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_sw_humanities_tasks
+task: global_mmlu_full_sw_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_prehistory.yaml
new file mode 100644
index 000000000..db94a2ff4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_sw_humanities_tasks
+task: global_mmlu_full_sw_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_accounting.yaml
new file mode 100644
index 000000000..7cd19d35d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_sw_other_tasks
+task: global_mmlu_full_sw_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_law.yaml
new file mode 100644
index 000000000..9434ae4c2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_sw_humanities_tasks
+task: global_mmlu_full_sw_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_medicine.yaml
new file mode 100644
index 000000000..cf35b9c64
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_sw_other_tasks
+task: global_mmlu_full_sw_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_psychology.yaml
new file mode 100644
index 000000000..7570e2889
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_sw_social_sciences_tasks
+task: global_mmlu_full_sw_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_public_relations.yaml
new file mode 100644
index 000000000..54c094db4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_sw_social_sciences_tasks
+task: global_mmlu_full_sw_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_security_studies.yaml
new file mode 100644
index 000000000..c8d5a42c0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_sw_social_sciences_tasks
+task: global_mmlu_full_sw_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_sociology.yaml
new file mode 100644
index 000000000..79d51a58d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_sw_social_sciences_tasks
+task: global_mmlu_full_sw_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_us_foreign_policy.yaml
new file mode 100644
index 000000000..523b1572b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_sw_social_sciences_tasks
+task: global_mmlu_full_sw_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_virology.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_virology.yaml
new file mode 100644
index 000000000..43179ff84
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_sw_other_tasks
+task: global_mmlu_full_sw_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_world_religions.yaml
new file mode 100644
index 000000000..bef7b7f84
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _sw_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_sw_humanities_tasks
+task: global_mmlu_full_sw_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/sw/utils.py b/lmms_eval/tasks/global_mmlu/full/sw/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/sw/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/te/_global_mmlu_full_te.yaml b/lmms_eval/tasks/global_mmlu/full/te/_global_mmlu_full_te.yaml
new file mode 100644
index 000000000..0bc967ded
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/_global_mmlu_full_te.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_te
+task:
+  - global_mmlu_full_te_stem
+  - global_mmlu_full_te_other
+  - global_mmlu_full_te_social_sciences
+  - global_mmlu_full_te_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/te/_global_mmlu_full_te_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/te/_global_mmlu_full_te_humanities.yaml
new file mode 100644
index 000000000..7a3c479e8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/_global_mmlu_full_te_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_te_humanities
+task:
+  - global_mmlu_full_te_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/te/_global_mmlu_full_te_other.yaml b/lmms_eval/tasks/global_mmlu/full/te/_global_mmlu_full_te_other.yaml
new file mode 100644
index 000000000..2932844ac
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/_global_mmlu_full_te_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_te_other
+task:
+  - global_mmlu_full_te_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/te/_global_mmlu_full_te_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/te/_global_mmlu_full_te_social_sciences.yaml
new file mode 100644
index 000000000..25e721db7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/_global_mmlu_full_te_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_te_social_sciences
+task:
+  - global_mmlu_full_te_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/te/_global_mmlu_full_te_stem.yaml b/lmms_eval/tasks/global_mmlu/full/te/_global_mmlu_full_te_stem.yaml
new file mode 100644
index 000000000..fe2426cac
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/_global_mmlu_full_te_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_te_stem
+task:
+  - global_mmlu_full_te_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/te/_te_template_yaml b/lmms_eval/tasks/global_mmlu/full/te/_te_template_yaml
new file mode 100644
index 000000000..d7b1190d4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/_te_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: te
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_abstract_algebra.yaml
new file mode 100644
index 000000000..e922fd08d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_te_stem_tasks
+task: global_mmlu_full_te_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_anatomy.yaml
new file mode 100644
index 000000000..00582018a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_te_stem_tasks
+task: global_mmlu_full_te_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_astronomy.yaml
new file mode 100644
index 000000000..5bc5e76ef
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_te_stem_tasks
+task: global_mmlu_full_te_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_business_ethics.yaml
new file mode 100644
index 000000000..7b440102c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_te_other_tasks
+task: global_mmlu_full_te_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_clinical_knowledge.yaml
new file mode 100644
index 000000000..90e561844
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_te_other_tasks
+task: global_mmlu_full_te_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_biology.yaml
new file mode 100644
index 000000000..0f036e60d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_te_stem_tasks
+task: global_mmlu_full_te_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_chemistry.yaml
new file mode 100644
index 000000000..ccdb849a8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_te_stem_tasks
+task: global_mmlu_full_te_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_computer_science.yaml
new file mode 100644
index 000000000..f11e5657b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_te_stem_tasks
+task: global_mmlu_full_te_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_mathematics.yaml
new file mode 100644
index 000000000..c5022ce29
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_te_stem_tasks
+task: global_mmlu_full_te_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_medicine.yaml
new file mode 100644
index 000000000..bd5219f07
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_te_other_tasks
+task: global_mmlu_full_te_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_physics.yaml
new file mode 100644
index 000000000..88dad05a2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_te_stem_tasks
+task: global_mmlu_full_te_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_computer_security.yaml
new file mode 100644
index 000000000..0e8f37fc1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_te_stem_tasks
+task: global_mmlu_full_te_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_conceptual_physics.yaml
new file mode 100644
index 000000000..f05276252
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_te_stem_tasks
+task: global_mmlu_full_te_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_econometrics.yaml
new file mode 100644
index 000000000..cf008a674
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_te_social_sciences_tasks
+task: global_mmlu_full_te_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_electrical_engineering.yaml
new file mode 100644
index 000000000..97169e931
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_te_stem_tasks
+task: global_mmlu_full_te_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_elementary_mathematics.yaml
new file mode 100644
index 000000000..f3edc8966
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_te_stem_tasks
+task: global_mmlu_full_te_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_formal_logic.yaml
new file mode 100644
index 000000000..d4c182d1d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_te_humanities_tasks
+task: global_mmlu_full_te_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_global_facts.yaml
new file mode 100644
index 000000000..53b52f4d3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_te_other_tasks
+task: global_mmlu_full_te_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_biology.yaml
new file mode 100644
index 000000000..5f02170ff
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_te_stem_tasks
+task: global_mmlu_full_te_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_chemistry.yaml
new file mode 100644
index 000000000..c77d30aa2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_te_stem_tasks
+task: global_mmlu_full_te_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_computer_science.yaml
new file mode 100644
index 000000000..7f388a060
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_te_stem_tasks
+task: global_mmlu_full_te_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_european_history.yaml
new file mode 100644
index 000000000..75d54d72b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_te_humanities_tasks
+task: global_mmlu_full_te_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_geography.yaml
new file mode 100644
index 000000000..383596ff0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_te_social_sciences_tasks
+task: global_mmlu_full_te_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..8db56a851
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_te_social_sciences_tasks
+task: global_mmlu_full_te_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..bd471b8d2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_te_social_sciences_tasks
+task: global_mmlu_full_te_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_mathematics.yaml
new file mode 100644
index 000000000..58f577ed1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_te_stem_tasks
+task: global_mmlu_full_te_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_microeconomics.yaml
new file mode 100644
index 000000000..400a3805f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_te_social_sciences_tasks
+task: global_mmlu_full_te_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_physics.yaml
new file mode 100644
index 000000000..694ddc304
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_te_stem_tasks
+task: global_mmlu_full_te_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_psychology.yaml
new file mode 100644
index 000000000..b900af19c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_te_social_sciences_tasks
+task: global_mmlu_full_te_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_statistics.yaml
new file mode 100644
index 000000000..3492e7249
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_te_stem_tasks
+task: global_mmlu_full_te_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_us_history.yaml
new file mode 100644
index 000000000..48a2d75a6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_te_humanities_tasks
+task: global_mmlu_full_te_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_world_history.yaml
new file mode 100644
index 000000000..7e95f7ea1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_te_humanities_tasks
+task: global_mmlu_full_te_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_human_aging.yaml
new file mode 100644
index 000000000..dc44c1b0a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_te_other_tasks
+task: global_mmlu_full_te_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_human_sexuality.yaml
new file mode 100644
index 000000000..d7631419b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_te_social_sciences_tasks
+task: global_mmlu_full_te_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_international_law.yaml
new file mode 100644
index 000000000..0c2c7862f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_te_humanities_tasks
+task: global_mmlu_full_te_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_jurisprudence.yaml
new file mode 100644
index 000000000..718cd9fad
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_te_humanities_tasks
+task: global_mmlu_full_te_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_logical_fallacies.yaml
new file mode 100644
index 000000000..7bb9170ce
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_te_humanities_tasks
+task: global_mmlu_full_te_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_machine_learning.yaml
new file mode 100644
index 000000000..123555383
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_te_stem_tasks
+task: global_mmlu_full_te_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_management.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_management.yaml
new file mode 100644
index 000000000..f092416ff
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_te_other_tasks
+task: global_mmlu_full_te_management
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_marketing.yaml
new file mode 100644
index 000000000..15b84b467
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_te_other_tasks
+task: global_mmlu_full_te_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_medical_genetics.yaml
new file mode 100644
index 000000000..8f0730be9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_te_other_tasks
+task: global_mmlu_full_te_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_miscellaneous.yaml
new file mode 100644
index 000000000..53487f558
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_te_other_tasks
+task: global_mmlu_full_te_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_moral_disputes.yaml
new file mode 100644
index 000000000..fca8df9b7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_te_humanities_tasks
+task: global_mmlu_full_te_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_moral_scenarios.yaml
new file mode 100644
index 000000000..d87f6b029
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_te_humanities_tasks
+task: global_mmlu_full_te_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_nutrition.yaml
new file mode 100644
index 000000000..9348a76e2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_te_other_tasks
+task: global_mmlu_full_te_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_philosophy.yaml
new file mode 100644
index 000000000..c8efe8d97
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_te_humanities_tasks
+task: global_mmlu_full_te_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_prehistory.yaml
new file mode 100644
index 000000000..b702542ea
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_te_humanities_tasks
+task: global_mmlu_full_te_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_accounting.yaml
new file mode 100644
index 000000000..045b6e1ca
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_te_other_tasks
+task: global_mmlu_full_te_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_law.yaml
new file mode 100644
index 000000000..5e5fa3080
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_te_humanities_tasks
+task: global_mmlu_full_te_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_medicine.yaml
new file mode 100644
index 000000000..d4ede33f0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_te_other_tasks
+task: global_mmlu_full_te_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_psychology.yaml
new file mode 100644
index 000000000..cb1906d41
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_te_social_sciences_tasks
+task: global_mmlu_full_te_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_public_relations.yaml
new file mode 100644
index 000000000..1ac09ce0d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_te_social_sciences_tasks
+task: global_mmlu_full_te_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_security_studies.yaml
new file mode 100644
index 000000000..bbb7bc7c8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_te_social_sciences_tasks
+task: global_mmlu_full_te_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_sociology.yaml
new file mode 100644
index 000000000..e080e0823
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_te_social_sciences_tasks
+task: global_mmlu_full_te_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_us_foreign_policy.yaml
new file mode 100644
index 000000000..338f08095
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_te_social_sciences_tasks
+task: global_mmlu_full_te_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_virology.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_virology.yaml
new file mode 100644
index 000000000..1f5e38a9e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_te_other_tasks
+task: global_mmlu_full_te_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_world_religions.yaml
new file mode 100644
index 000000000..4da26e3e1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _te_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_te_humanities_tasks
+task: global_mmlu_full_te_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/te/utils.py b/lmms_eval/tasks/global_mmlu/full/te/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/te/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr.yaml b/lmms_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr.yaml
new file mode 100644
index 000000000..aa4ae63fa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_tr
+task:
+  - global_mmlu_full_tr_stem
+  - global_mmlu_full_tr_other
+  - global_mmlu_full_tr_social_sciences
+  - global_mmlu_full_tr_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_humanities.yaml
new file mode 100644
index 000000000..f4dade158
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_tr_humanities
+task:
+  - global_mmlu_full_tr_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_other.yaml b/lmms_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_other.yaml
new file mode 100644
index 000000000..e80a5b9d8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_tr_other
+task:
+  - global_mmlu_full_tr_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_social_sciences.yaml
new file mode 100644
index 000000000..56fc20e14
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_tr_social_sciences
+task:
+  - global_mmlu_full_tr_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_stem.yaml b/lmms_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_stem.yaml
new file mode 100644
index 000000000..51f9bb3d9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_tr_stem
+task:
+  - global_mmlu_full_tr_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/_tr_template_yaml b/lmms_eval/tasks/global_mmlu/full/tr/_tr_template_yaml
new file mode 100644
index 000000000..e322bee67
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/_tr_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: tr
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_abstract_algebra.yaml
new file mode 100644
index 000000000..1e8215730
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_tr_stem_tasks
+task: global_mmlu_full_tr_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_anatomy.yaml
new file mode 100644
index 000000000..444402254
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_tr_stem_tasks
+task: global_mmlu_full_tr_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_astronomy.yaml
new file mode 100644
index 000000000..e85390bf1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_tr_stem_tasks
+task: global_mmlu_full_tr_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_business_ethics.yaml
new file mode 100644
index 000000000..4b1afc9cf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_tr_other_tasks
+task: global_mmlu_full_tr_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_clinical_knowledge.yaml
new file mode 100644
index 000000000..bdfa69e6d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_tr_other_tasks
+task: global_mmlu_full_tr_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_biology.yaml
new file mode 100644
index 000000000..df43a67ce
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_tr_stem_tasks
+task: global_mmlu_full_tr_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_chemistry.yaml
new file mode 100644
index 000000000..af2b8b3e8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_tr_stem_tasks
+task: global_mmlu_full_tr_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_computer_science.yaml
new file mode 100644
index 000000000..622854f4d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_tr_stem_tasks
+task: global_mmlu_full_tr_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_mathematics.yaml
new file mode 100644
index 000000000..902bd9c1b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_tr_stem_tasks
+task: global_mmlu_full_tr_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_medicine.yaml
new file mode 100644
index 000000000..6b44d0d13
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_tr_other_tasks
+task: global_mmlu_full_tr_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_physics.yaml
new file mode 100644
index 000000000..27540d97f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_tr_stem_tasks
+task: global_mmlu_full_tr_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_computer_security.yaml
new file mode 100644
index 000000000..dbcabeed0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_tr_stem_tasks
+task: global_mmlu_full_tr_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_conceptual_physics.yaml
new file mode 100644
index 000000000..628a4fcf0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_tr_stem_tasks
+task: global_mmlu_full_tr_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_econometrics.yaml
new file mode 100644
index 000000000..6feb236f9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_tr_social_sciences_tasks
+task: global_mmlu_full_tr_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_electrical_engineering.yaml
new file mode 100644
index 000000000..9a2a8665c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_tr_stem_tasks
+task: global_mmlu_full_tr_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_elementary_mathematics.yaml
new file mode 100644
index 000000000..ffc6dee79
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_tr_stem_tasks
+task: global_mmlu_full_tr_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_formal_logic.yaml
new file mode 100644
index 000000000..77c189a0b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_tr_humanities_tasks
+task: global_mmlu_full_tr_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_global_facts.yaml
new file mode 100644
index 000000000..a756d102e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_tr_other_tasks
+task: global_mmlu_full_tr_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_biology.yaml
new file mode 100644
index 000000000..51e7dd9e9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_tr_stem_tasks
+task: global_mmlu_full_tr_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_chemistry.yaml
new file mode 100644
index 000000000..077476ae7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_tr_stem_tasks
+task: global_mmlu_full_tr_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_computer_science.yaml
new file mode 100644
index 000000000..cb60e0425
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_tr_stem_tasks
+task: global_mmlu_full_tr_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_european_history.yaml
new file mode 100644
index 000000000..2b989e058
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_tr_humanities_tasks
+task: global_mmlu_full_tr_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_geography.yaml
new file mode 100644
index 000000000..8a0c4d900
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_tr_social_sciences_tasks
+task: global_mmlu_full_tr_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..2a585f02d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_tr_social_sciences_tasks
+task: global_mmlu_full_tr_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..f88e98310
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_tr_social_sciences_tasks
+task: global_mmlu_full_tr_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_mathematics.yaml
new file mode 100644
index 000000000..e880b0b53
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_tr_stem_tasks
+task: global_mmlu_full_tr_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_microeconomics.yaml
new file mode 100644
index 000000000..5527bed2c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_tr_social_sciences_tasks
+task: global_mmlu_full_tr_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_physics.yaml
new file mode 100644
index 000000000..da93a96e1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_tr_stem_tasks
+task: global_mmlu_full_tr_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_psychology.yaml
new file mode 100644
index 000000000..a28e110c3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_tr_social_sciences_tasks
+task: global_mmlu_full_tr_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_statistics.yaml
new file mode 100644
index 000000000..93871dcf8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_tr_stem_tasks
+task: global_mmlu_full_tr_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_us_history.yaml
new file mode 100644
index 000000000..507a4d5ce
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_tr_humanities_tasks
+task: global_mmlu_full_tr_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_world_history.yaml
new file mode 100644
index 000000000..60cc713ef
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_tr_humanities_tasks
+task: global_mmlu_full_tr_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_human_aging.yaml
new file mode 100644
index 000000000..8e48bf122
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_tr_other_tasks
+task: global_mmlu_full_tr_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_human_sexuality.yaml
new file mode 100644
index 000000000..84a958509
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_tr_social_sciences_tasks
+task: global_mmlu_full_tr_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_international_law.yaml
new file mode 100644
index 000000000..d0dc429f0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_tr_humanities_tasks
+task: global_mmlu_full_tr_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_jurisprudence.yaml
new file mode 100644
index 000000000..ea3b7a515
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_tr_humanities_tasks
+task: global_mmlu_full_tr_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_logical_fallacies.yaml
new file mode 100644
index 000000000..cd61d7d74
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_tr_humanities_tasks
+task: global_mmlu_full_tr_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_machine_learning.yaml
new file mode 100644
index 000000000..b0e785c33
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_tr_stem_tasks
+task: global_mmlu_full_tr_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_management.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_management.yaml
new file mode 100644
index 000000000..5ce0d753b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_tr_other_tasks
+task: global_mmlu_full_tr_management
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_marketing.yaml
new file mode 100644
index 000000000..8ffd4986f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_tr_other_tasks
+task: global_mmlu_full_tr_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_medical_genetics.yaml
new file mode 100644
index 000000000..43814b40c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_tr_other_tasks
+task: global_mmlu_full_tr_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_miscellaneous.yaml
new file mode 100644
index 000000000..e21cfcf61
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_tr_other_tasks
+task: global_mmlu_full_tr_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_moral_disputes.yaml
new file mode 100644
index 000000000..88fbfbe2c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_tr_humanities_tasks
+task: global_mmlu_full_tr_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_moral_scenarios.yaml
new file mode 100644
index 000000000..9f92f855d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_tr_humanities_tasks
+task: global_mmlu_full_tr_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_nutrition.yaml
new file mode 100644
index 000000000..31b39c386
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_tr_other_tasks
+task: global_mmlu_full_tr_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_philosophy.yaml
new file mode 100644
index 000000000..283a2b896
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_tr_humanities_tasks
+task: global_mmlu_full_tr_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_prehistory.yaml
new file mode 100644
index 000000000..e4c170145
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_tr_humanities_tasks
+task: global_mmlu_full_tr_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_accounting.yaml
new file mode 100644
index 000000000..c69f14f76
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_tr_other_tasks
+task: global_mmlu_full_tr_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_law.yaml
new file mode 100644
index 000000000..8f5e97c69
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_tr_humanities_tasks
+task: global_mmlu_full_tr_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_medicine.yaml
new file mode 100644
index 000000000..00a5f32aa
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_tr_other_tasks
+task: global_mmlu_full_tr_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_psychology.yaml
new file mode 100644
index 000000000..c8571bdb0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_tr_social_sciences_tasks
+task: global_mmlu_full_tr_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_public_relations.yaml
new file mode 100644
index 000000000..539f8da66
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_tr_social_sciences_tasks
+task: global_mmlu_full_tr_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_security_studies.yaml
new file mode 100644
index 000000000..4203e3659
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_tr_social_sciences_tasks
+task: global_mmlu_full_tr_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_sociology.yaml
new file mode 100644
index 000000000..9cf6352c4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_tr_social_sciences_tasks
+task: global_mmlu_full_tr_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_us_foreign_policy.yaml
new file mode 100644
index 000000000..b86a699b6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_tr_social_sciences_tasks
+task: global_mmlu_full_tr_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_virology.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_virology.yaml
new file mode 100644
index 000000000..001cbb288
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_tr_other_tasks
+task: global_mmlu_full_tr_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_world_religions.yaml
new file mode 100644
index 000000000..1f1d4e4f1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _tr_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_tr_humanities_tasks
+task: global_mmlu_full_tr_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/tr/utils.py b/lmms_eval/tasks/global_mmlu/full/tr/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/tr/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/_global_mmlu_full_uk.yaml b/lmms_eval/tasks/global_mmlu/full/uk/_global_mmlu_full_uk.yaml
new file mode 100644
index 000000000..9bbd14cb8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/_global_mmlu_full_uk.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_uk
+task:
+  - global_mmlu_full_uk_stem
+  - global_mmlu_full_uk_other
+  - global_mmlu_full_uk_social_sciences
+  - global_mmlu_full_uk_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_humanities.yaml
new file mode 100644
index 000000000..b3ec01db6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_uk_humanities
+task:
+  - global_mmlu_full_uk_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_other.yaml b/lmms_eval/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_other.yaml
new file mode 100644
index 000000000..176b18617
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_uk_other
+task:
+  - global_mmlu_full_uk_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_social_sciences.yaml
new file mode 100644
index 000000000..66b36a608
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_uk_social_sciences
+task:
+  - global_mmlu_full_uk_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_stem.yaml b/lmms_eval/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_stem.yaml
new file mode 100644
index 000000000..4deba6571
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_uk_stem
+task:
+  - global_mmlu_full_uk_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/_uk_template_yaml b/lmms_eval/tasks/global_mmlu/full/uk/_uk_template_yaml
new file mode 100644
index 000000000..5765ce13a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/_uk_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: uk
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_abstract_algebra.yaml
new file mode 100644
index 000000000..ce37c7153
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_uk_stem_tasks
+task: global_mmlu_full_uk_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_anatomy.yaml
new file mode 100644
index 000000000..db1433d5c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_uk_stem_tasks
+task: global_mmlu_full_uk_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_astronomy.yaml
new file mode 100644
index 000000000..6b123eced
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_uk_stem_tasks
+task: global_mmlu_full_uk_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_business_ethics.yaml
new file mode 100644
index 000000000..775d2f2c4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_uk_other_tasks
+task: global_mmlu_full_uk_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_clinical_knowledge.yaml
new file mode 100644
index 000000000..5f71076df
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_uk_other_tasks
+task: global_mmlu_full_uk_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_biology.yaml
new file mode 100644
index 000000000..92342ac7c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_uk_stem_tasks
+task: global_mmlu_full_uk_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_chemistry.yaml
new file mode 100644
index 000000000..71384a8b5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_uk_stem_tasks
+task: global_mmlu_full_uk_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_computer_science.yaml
new file mode 100644
index 000000000..6013afe17
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_uk_stem_tasks
+task: global_mmlu_full_uk_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_mathematics.yaml
new file mode 100644
index 000000000..27b604915
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_uk_stem_tasks
+task: global_mmlu_full_uk_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_medicine.yaml
new file mode 100644
index 000000000..87131c250
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_uk_other_tasks
+task: global_mmlu_full_uk_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_physics.yaml
new file mode 100644
index 000000000..93109632c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_uk_stem_tasks
+task: global_mmlu_full_uk_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_computer_security.yaml
new file mode 100644
index 000000000..0f11fcce9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_uk_stem_tasks
+task: global_mmlu_full_uk_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_conceptual_physics.yaml
new file mode 100644
index 000000000..7ff9715a0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_uk_stem_tasks
+task: global_mmlu_full_uk_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_econometrics.yaml
new file mode 100644
index 000000000..ba92e4b8c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_uk_social_sciences_tasks
+task: global_mmlu_full_uk_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_electrical_engineering.yaml
new file mode 100644
index 000000000..3a1c86ffd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_uk_stem_tasks
+task: global_mmlu_full_uk_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_elementary_mathematics.yaml
new file mode 100644
index 000000000..7d80cce7c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_uk_stem_tasks
+task: global_mmlu_full_uk_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_formal_logic.yaml
new file mode 100644
index 000000000..9f8a40912
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_uk_humanities_tasks
+task: global_mmlu_full_uk_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_global_facts.yaml
new file mode 100644
index 000000000..ebd6c2da9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_uk_other_tasks
+task: global_mmlu_full_uk_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_biology.yaml
new file mode 100644
index 000000000..a8b0cf3a8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_uk_stem_tasks
+task: global_mmlu_full_uk_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_chemistry.yaml
new file mode 100644
index 000000000..010dbec34
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_uk_stem_tasks
+task: global_mmlu_full_uk_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_computer_science.yaml
new file mode 100644
index 000000000..9a2701440
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_uk_stem_tasks
+task: global_mmlu_full_uk_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_european_history.yaml
new file mode 100644
index 000000000..52e80017c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_uk_humanities_tasks
+task: global_mmlu_full_uk_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_geography.yaml
new file mode 100644
index 000000000..4f41dd3d8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_uk_social_sciences_tasks
+task: global_mmlu_full_uk_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..72c589ef3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_uk_social_sciences_tasks
+task: global_mmlu_full_uk_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..e70675d9a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_uk_social_sciences_tasks
+task: global_mmlu_full_uk_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_mathematics.yaml
new file mode 100644
index 000000000..e29c558ea
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_uk_stem_tasks
+task: global_mmlu_full_uk_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_microeconomics.yaml
new file mode 100644
index 000000000..6b7354950
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_uk_social_sciences_tasks
+task: global_mmlu_full_uk_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_physics.yaml
new file mode 100644
index 000000000..69a03c065
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_uk_stem_tasks
+task: global_mmlu_full_uk_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_psychology.yaml
new file mode 100644
index 000000000..9b02711c2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_uk_social_sciences_tasks
+task: global_mmlu_full_uk_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_statistics.yaml
new file mode 100644
index 000000000..60cc0cdda
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_uk_stem_tasks
+task: global_mmlu_full_uk_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_us_history.yaml
new file mode 100644
index 000000000..b62244ebb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_uk_humanities_tasks
+task: global_mmlu_full_uk_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_world_history.yaml
new file mode 100644
index 000000000..57667edc2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_uk_humanities_tasks
+task: global_mmlu_full_uk_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_human_aging.yaml
new file mode 100644
index 000000000..028048904
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_uk_other_tasks
+task: global_mmlu_full_uk_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_human_sexuality.yaml
new file mode 100644
index 000000000..37382bab0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_uk_social_sciences_tasks
+task: global_mmlu_full_uk_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_international_law.yaml
new file mode 100644
index 000000000..d1b046d79
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_uk_humanities_tasks
+task: global_mmlu_full_uk_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_jurisprudence.yaml
new file mode 100644
index 000000000..12b9da523
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_uk_humanities_tasks
+task: global_mmlu_full_uk_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_logical_fallacies.yaml
new file mode 100644
index 000000000..abb2de2a0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_uk_humanities_tasks
+task: global_mmlu_full_uk_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_machine_learning.yaml
new file mode 100644
index 000000000..7a1a6f349
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_uk_stem_tasks
+task: global_mmlu_full_uk_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_management.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_management.yaml
new file mode 100644
index 000000000..ec4cb17d3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_uk_other_tasks
+task: global_mmlu_full_uk_management
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_marketing.yaml
new file mode 100644
index 000000000..afbdaee2e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_uk_other_tasks
+task: global_mmlu_full_uk_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_medical_genetics.yaml
new file mode 100644
index 000000000..bc1fe1bd1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_uk_other_tasks
+task: global_mmlu_full_uk_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_miscellaneous.yaml
new file mode 100644
index 000000000..8f3b18f83
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_uk_other_tasks
+task: global_mmlu_full_uk_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_moral_disputes.yaml
new file mode 100644
index 000000000..34b54e34d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_uk_humanities_tasks
+task: global_mmlu_full_uk_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_moral_scenarios.yaml
new file mode 100644
index 000000000..387069776
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_uk_humanities_tasks
+task: global_mmlu_full_uk_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_nutrition.yaml
new file mode 100644
index 000000000..9f9dd1fe8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_uk_other_tasks
+task: global_mmlu_full_uk_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_philosophy.yaml
new file mode 100644
index 000000000..4e9810083
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_uk_humanities_tasks
+task: global_mmlu_full_uk_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_prehistory.yaml
new file mode 100644
index 000000000..08e3c2af2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_uk_humanities_tasks
+task: global_mmlu_full_uk_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_accounting.yaml
new file mode 100644
index 000000000..dc02a7b26
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_uk_other_tasks
+task: global_mmlu_full_uk_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_law.yaml
new file mode 100644
index 000000000..7090a6e1d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_uk_humanities_tasks
+task: global_mmlu_full_uk_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_medicine.yaml
new file mode 100644
index 000000000..0b43dcfbc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_uk_other_tasks
+task: global_mmlu_full_uk_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_psychology.yaml
new file mode 100644
index 000000000..b279a94c4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_uk_social_sciences_tasks
+task: global_mmlu_full_uk_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_public_relations.yaml
new file mode 100644
index 000000000..3b45dc621
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_uk_social_sciences_tasks
+task: global_mmlu_full_uk_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_security_studies.yaml
new file mode 100644
index 000000000..4ea308da2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_uk_social_sciences_tasks
+task: global_mmlu_full_uk_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_sociology.yaml
new file mode 100644
index 000000000..a7aa08ec0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_uk_social_sciences_tasks
+task: global_mmlu_full_uk_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_us_foreign_policy.yaml
new file mode 100644
index 000000000..d089e7783
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_uk_social_sciences_tasks
+task: global_mmlu_full_uk_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_virology.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_virology.yaml
new file mode 100644
index 000000000..41b627f5a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_uk_other_tasks
+task: global_mmlu_full_uk_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_world_religions.yaml
new file mode 100644
index 000000000..f5d6d4151
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/global_mmlu_full_uk_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _uk_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_uk_humanities_tasks
+task: global_mmlu_full_uk_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/uk/utils.py b/lmms_eval/tasks/global_mmlu/full/uk/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/uk/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/_global_mmlu_full_vi.yaml b/lmms_eval/tasks/global_mmlu/full/vi/_global_mmlu_full_vi.yaml
new file mode 100644
index 000000000..278884741
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/_global_mmlu_full_vi.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_vi
+task:
+  - global_mmlu_full_vi_stem
+  - global_mmlu_full_vi_other
+  - global_mmlu_full_vi_social_sciences
+  - global_mmlu_full_vi_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_humanities.yaml
new file mode 100644
index 000000000..7a05accaf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_vi_humanities
+task:
+  - global_mmlu_full_vi_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_other.yaml b/lmms_eval/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_other.yaml
new file mode 100644
index 000000000..880bab9a3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_vi_other
+task:
+  - global_mmlu_full_vi_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_social_sciences.yaml
new file mode 100644
index 000000000..6da224f2a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_vi_social_sciences
+task:
+  - global_mmlu_full_vi_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_stem.yaml b/lmms_eval/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_stem.yaml
new file mode 100644
index 000000000..12526ce7a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_vi_stem
+task:
+  - global_mmlu_full_vi_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/_vi_template_yaml b/lmms_eval/tasks/global_mmlu/full/vi/_vi_template_yaml
new file mode 100644
index 000000000..5a0ca8175
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/_vi_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: vi
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_abstract_algebra.yaml
new file mode 100644
index 000000000..47dc80cee
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_vi_stem_tasks
+task: global_mmlu_full_vi_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_anatomy.yaml
new file mode 100644
index 000000000..d29cb5836
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_vi_stem_tasks
+task: global_mmlu_full_vi_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_astronomy.yaml
new file mode 100644
index 000000000..3e3ba1dc2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_vi_stem_tasks
+task: global_mmlu_full_vi_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_business_ethics.yaml
new file mode 100644
index 000000000..3afecdc13
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_vi_other_tasks
+task: global_mmlu_full_vi_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_clinical_knowledge.yaml
new file mode 100644
index 000000000..34a90a8ed
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_vi_other_tasks
+task: global_mmlu_full_vi_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_biology.yaml
new file mode 100644
index 000000000..63a4c7726
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_vi_stem_tasks
+task: global_mmlu_full_vi_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_chemistry.yaml
new file mode 100644
index 000000000..f7226e020
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_vi_stem_tasks
+task: global_mmlu_full_vi_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_computer_science.yaml
new file mode 100644
index 000000000..90a9e0b2c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_vi_stem_tasks
+task: global_mmlu_full_vi_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_mathematics.yaml
new file mode 100644
index 000000000..a09173d6a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_vi_stem_tasks
+task: global_mmlu_full_vi_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_medicine.yaml
new file mode 100644
index 000000000..22dc78bdf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_vi_other_tasks
+task: global_mmlu_full_vi_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_physics.yaml
new file mode 100644
index 000000000..a6f8dbca3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_vi_stem_tasks
+task: global_mmlu_full_vi_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_computer_security.yaml
new file mode 100644
index 000000000..4d4b3d605
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_vi_stem_tasks
+task: global_mmlu_full_vi_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_conceptual_physics.yaml
new file mode 100644
index 000000000..6c501d0a3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_vi_stem_tasks
+task: global_mmlu_full_vi_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_econometrics.yaml
new file mode 100644
index 000000000..d0936b3be
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_vi_social_sciences_tasks
+task: global_mmlu_full_vi_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_electrical_engineering.yaml
new file mode 100644
index 000000000..3b23387f8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_vi_stem_tasks
+task: global_mmlu_full_vi_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_elementary_mathematics.yaml
new file mode 100644
index 000000000..9c0982663
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_vi_stem_tasks
+task: global_mmlu_full_vi_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_formal_logic.yaml
new file mode 100644
index 000000000..21a28bb4b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_vi_humanities_tasks
+task: global_mmlu_full_vi_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_global_facts.yaml
new file mode 100644
index 000000000..a912dba1b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_vi_other_tasks
+task: global_mmlu_full_vi_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_biology.yaml
new file mode 100644
index 000000000..e334fb1c9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_vi_stem_tasks
+task: global_mmlu_full_vi_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_chemistry.yaml
new file mode 100644
index 000000000..ba98297eb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_vi_stem_tasks
+task: global_mmlu_full_vi_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_computer_science.yaml
new file mode 100644
index 000000000..22e0b00a3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_vi_stem_tasks
+task: global_mmlu_full_vi_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_european_history.yaml
new file mode 100644
index 000000000..06507b7cc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_vi_humanities_tasks
+task: global_mmlu_full_vi_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_geography.yaml
new file mode 100644
index 000000000..d6eeec7ad
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_vi_social_sciences_tasks
+task: global_mmlu_full_vi_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..2faf2b09e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_vi_social_sciences_tasks
+task: global_mmlu_full_vi_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..16ed50b8c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_vi_social_sciences_tasks
+task: global_mmlu_full_vi_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_mathematics.yaml
new file mode 100644
index 000000000..1cad75ecb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_vi_stem_tasks
+task: global_mmlu_full_vi_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_microeconomics.yaml
new file mode 100644
index 000000000..4499711fd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_vi_social_sciences_tasks
+task: global_mmlu_full_vi_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_physics.yaml
new file mode 100644
index 000000000..bb92f446d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_vi_stem_tasks
+task: global_mmlu_full_vi_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_psychology.yaml
new file mode 100644
index 000000000..0a12e4deb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_vi_social_sciences_tasks
+task: global_mmlu_full_vi_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_statistics.yaml
new file mode 100644
index 000000000..3ae34e4de
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_vi_stem_tasks
+task: global_mmlu_full_vi_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_us_history.yaml
new file mode 100644
index 000000000..9ad96b12d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_vi_humanities_tasks
+task: global_mmlu_full_vi_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_world_history.yaml
new file mode 100644
index 000000000..5df3661c7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_vi_humanities_tasks
+task: global_mmlu_full_vi_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_human_aging.yaml
new file mode 100644
index 000000000..57820fab4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_vi_other_tasks
+task: global_mmlu_full_vi_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_human_sexuality.yaml
new file mode 100644
index 000000000..5b53962b1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_vi_social_sciences_tasks
+task: global_mmlu_full_vi_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_international_law.yaml
new file mode 100644
index 000000000..5f81b09e5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_vi_humanities_tasks
+task: global_mmlu_full_vi_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_jurisprudence.yaml
new file mode 100644
index 000000000..52ec47d4f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_vi_humanities_tasks
+task: global_mmlu_full_vi_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_logical_fallacies.yaml
new file mode 100644
index 000000000..ed89994d5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_vi_humanities_tasks
+task: global_mmlu_full_vi_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_machine_learning.yaml
new file mode 100644
index 000000000..258bd8c46
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_vi_stem_tasks
+task: global_mmlu_full_vi_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_management.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_management.yaml
new file mode 100644
index 000000000..1bd2f606e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_vi_other_tasks
+task: global_mmlu_full_vi_management
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_marketing.yaml
new file mode 100644
index 000000000..951a36422
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_vi_other_tasks
+task: global_mmlu_full_vi_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_medical_genetics.yaml
new file mode 100644
index 000000000..9d6060070
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_vi_other_tasks
+task: global_mmlu_full_vi_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_miscellaneous.yaml
new file mode 100644
index 000000000..a0cae1b03
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_vi_other_tasks
+task: global_mmlu_full_vi_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_moral_disputes.yaml
new file mode 100644
index 000000000..07987487c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_vi_humanities_tasks
+task: global_mmlu_full_vi_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_moral_scenarios.yaml
new file mode 100644
index 000000000..6a852bc65
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_vi_humanities_tasks
+task: global_mmlu_full_vi_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_nutrition.yaml
new file mode 100644
index 000000000..42b198f3f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_vi_other_tasks
+task: global_mmlu_full_vi_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_philosophy.yaml
new file mode 100644
index 000000000..a7ffc3161
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_vi_humanities_tasks
+task: global_mmlu_full_vi_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_prehistory.yaml
new file mode 100644
index 000000000..963496749
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_vi_humanities_tasks
+task: global_mmlu_full_vi_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_accounting.yaml
new file mode 100644
index 000000000..da949e34f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_vi_other_tasks
+task: global_mmlu_full_vi_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_law.yaml
new file mode 100644
index 000000000..81c74535e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_vi_humanities_tasks
+task: global_mmlu_full_vi_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_medicine.yaml
new file mode 100644
index 000000000..7315b3535
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_vi_other_tasks
+task: global_mmlu_full_vi_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_psychology.yaml
new file mode 100644
index 000000000..f2eb16528
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_vi_social_sciences_tasks
+task: global_mmlu_full_vi_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_public_relations.yaml
new file mode 100644
index 000000000..12933f08e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_vi_social_sciences_tasks
+task: global_mmlu_full_vi_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_security_studies.yaml
new file mode 100644
index 000000000..7e90ba554
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_vi_social_sciences_tasks
+task: global_mmlu_full_vi_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_sociology.yaml
new file mode 100644
index 000000000..056c757b8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_vi_social_sciences_tasks
+task: global_mmlu_full_vi_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_us_foreign_policy.yaml
new file mode 100644
index 000000000..5bcd95d61
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_vi_social_sciences_tasks
+task: global_mmlu_full_vi_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_virology.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_virology.yaml
new file mode 100644
index 000000000..775b0cca8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_vi_other_tasks
+task: global_mmlu_full_vi_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_world_religions.yaml
new file mode 100644
index 000000000..db6ba6e02
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/global_mmlu_full_vi_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _vi_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_vi_humanities_tasks
+task: global_mmlu_full_vi_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/vi/utils.py b/lmms_eval/tasks/global_mmlu/full/vi/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/vi/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/_global_mmlu_full_yo.yaml b/lmms_eval/tasks/global_mmlu/full/yo/_global_mmlu_full_yo.yaml
new file mode 100644
index 000000000..ef4930b6e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/_global_mmlu_full_yo.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_yo
+task:
+  - global_mmlu_full_yo_stem
+  - global_mmlu_full_yo_other
+  - global_mmlu_full_yo_social_sciences
+  - global_mmlu_full_yo_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_humanities.yaml
new file mode 100644
index 000000000..4e3b3c119
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_yo_humanities
+task:
+  - global_mmlu_full_yo_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_other.yaml b/lmms_eval/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_other.yaml
new file mode 100644
index 000000000..ed81bdfec
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_yo_other
+task:
+  - global_mmlu_full_yo_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_social_sciences.yaml
new file mode 100644
index 000000000..bab52fa22
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_yo_social_sciences
+task:
+  - global_mmlu_full_yo_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_stem.yaml b/lmms_eval/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_stem.yaml
new file mode 100644
index 000000000..3687d569e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_yo_stem
+task:
+  - global_mmlu_full_yo_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/_yo_template_yaml b/lmms_eval/tasks/global_mmlu/full/yo/_yo_template_yaml
new file mode 100644
index 000000000..ceefadf5b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/_yo_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: yo
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_abstract_algebra.yaml
new file mode 100644
index 000000000..ef817a386
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_yo_stem_tasks
+task: global_mmlu_full_yo_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_anatomy.yaml
new file mode 100644
index 000000000..a3bae5d55
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_yo_stem_tasks
+task: global_mmlu_full_yo_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_astronomy.yaml
new file mode 100644
index 000000000..b39aa143c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_yo_stem_tasks
+task: global_mmlu_full_yo_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_business_ethics.yaml
new file mode 100644
index 000000000..588329820
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_yo_other_tasks
+task: global_mmlu_full_yo_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_clinical_knowledge.yaml
new file mode 100644
index 000000000..21dcf8421
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_yo_other_tasks
+task: global_mmlu_full_yo_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_biology.yaml
new file mode 100644
index 000000000..f3abaf24c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_yo_stem_tasks
+task: global_mmlu_full_yo_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_chemistry.yaml
new file mode 100644
index 000000000..0468634b0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_yo_stem_tasks
+task: global_mmlu_full_yo_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_computer_science.yaml
new file mode 100644
index 000000000..df6e5844e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_yo_stem_tasks
+task: global_mmlu_full_yo_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_mathematics.yaml
new file mode 100644
index 000000000..0542a4fe2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_yo_stem_tasks
+task: global_mmlu_full_yo_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_medicine.yaml
new file mode 100644
index 000000000..cce0b497d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_yo_other_tasks
+task: global_mmlu_full_yo_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_physics.yaml
new file mode 100644
index 000000000..84ca1413e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_yo_stem_tasks
+task: global_mmlu_full_yo_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_computer_security.yaml
new file mode 100644
index 000000000..001689e93
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_yo_stem_tasks
+task: global_mmlu_full_yo_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_conceptual_physics.yaml
new file mode 100644
index 000000000..dcff962c3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_yo_stem_tasks
+task: global_mmlu_full_yo_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_econometrics.yaml
new file mode 100644
index 000000000..6d055d6d5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_yo_social_sciences_tasks
+task: global_mmlu_full_yo_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_electrical_engineering.yaml
new file mode 100644
index 000000000..c21f7f022
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_yo_stem_tasks
+task: global_mmlu_full_yo_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_elementary_mathematics.yaml
new file mode 100644
index 000000000..9b6173f0d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_yo_stem_tasks
+task: global_mmlu_full_yo_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_formal_logic.yaml
new file mode 100644
index 000000000..2ffc9740c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_yo_humanities_tasks
+task: global_mmlu_full_yo_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_global_facts.yaml
new file mode 100644
index 000000000..394a143ac
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_yo_other_tasks
+task: global_mmlu_full_yo_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_biology.yaml
new file mode 100644
index 000000000..f0de18874
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_yo_stem_tasks
+task: global_mmlu_full_yo_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_chemistry.yaml
new file mode 100644
index 000000000..02b16faec
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_yo_stem_tasks
+task: global_mmlu_full_yo_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_computer_science.yaml
new file mode 100644
index 000000000..94733faa0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_yo_stem_tasks
+task: global_mmlu_full_yo_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_european_history.yaml
new file mode 100644
index 000000000..6ec4070e8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_yo_humanities_tasks
+task: global_mmlu_full_yo_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_geography.yaml
new file mode 100644
index 000000000..4ab051d9b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_yo_social_sciences_tasks
+task: global_mmlu_full_yo_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..bedf7f204
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_yo_social_sciences_tasks
+task: global_mmlu_full_yo_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..cb4867094
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_yo_social_sciences_tasks
+task: global_mmlu_full_yo_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_mathematics.yaml
new file mode 100644
index 000000000..cea21a893
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_yo_stem_tasks
+task: global_mmlu_full_yo_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_microeconomics.yaml
new file mode 100644
index 000000000..a8eae6cdd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_yo_social_sciences_tasks
+task: global_mmlu_full_yo_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_physics.yaml
new file mode 100644
index 000000000..cdaca54fd
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_yo_stem_tasks
+task: global_mmlu_full_yo_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_psychology.yaml
new file mode 100644
index 000000000..ef3d75271
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_yo_social_sciences_tasks
+task: global_mmlu_full_yo_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_statistics.yaml
new file mode 100644
index 000000000..0ec62db01
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_yo_stem_tasks
+task: global_mmlu_full_yo_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_us_history.yaml
new file mode 100644
index 000000000..30c8573c4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_yo_humanities_tasks
+task: global_mmlu_full_yo_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_world_history.yaml
new file mode 100644
index 000000000..52f91d43b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_yo_humanities_tasks
+task: global_mmlu_full_yo_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_human_aging.yaml
new file mode 100644
index 000000000..4ab0ec2b9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_yo_other_tasks
+task: global_mmlu_full_yo_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_human_sexuality.yaml
new file mode 100644
index 000000000..f510c2d10
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_yo_social_sciences_tasks
+task: global_mmlu_full_yo_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_international_law.yaml
new file mode 100644
index 000000000..9b657110a
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_yo_humanities_tasks
+task: global_mmlu_full_yo_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_jurisprudence.yaml
new file mode 100644
index 000000000..e3ac0a521
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_yo_humanities_tasks
+task: global_mmlu_full_yo_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_logical_fallacies.yaml
new file mode 100644
index 000000000..a7a9e7186
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_yo_humanities_tasks
+task: global_mmlu_full_yo_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_machine_learning.yaml
new file mode 100644
index 000000000..4a61d3ae9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_yo_stem_tasks
+task: global_mmlu_full_yo_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_management.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_management.yaml
new file mode 100644
index 000000000..92b0b526d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_yo_other_tasks
+task: global_mmlu_full_yo_management
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_marketing.yaml
new file mode 100644
index 000000000..74c17559d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_yo_other_tasks
+task: global_mmlu_full_yo_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_medical_genetics.yaml
new file mode 100644
index 000000000..cfc2c8cbc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_yo_other_tasks
+task: global_mmlu_full_yo_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_miscellaneous.yaml
new file mode 100644
index 000000000..ad12bde64
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_yo_other_tasks
+task: global_mmlu_full_yo_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_moral_disputes.yaml
new file mode 100644
index 000000000..2e85331c9
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_yo_humanities_tasks
+task: global_mmlu_full_yo_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_moral_scenarios.yaml
new file mode 100644
index 000000000..9a6a6fc6d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_yo_humanities_tasks
+task: global_mmlu_full_yo_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_nutrition.yaml
new file mode 100644
index 000000000..62d9ae7bb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_yo_other_tasks
+task: global_mmlu_full_yo_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_philosophy.yaml
new file mode 100644
index 000000000..de42ec7a8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_yo_humanities_tasks
+task: global_mmlu_full_yo_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_prehistory.yaml
new file mode 100644
index 000000000..e2ad32362
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_yo_humanities_tasks
+task: global_mmlu_full_yo_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_accounting.yaml
new file mode 100644
index 000000000..198f227b8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_yo_other_tasks
+task: global_mmlu_full_yo_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_law.yaml
new file mode 100644
index 000000000..e5942f74b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_yo_humanities_tasks
+task: global_mmlu_full_yo_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_medicine.yaml
new file mode 100644
index 000000000..efd4ab7df
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_yo_other_tasks
+task: global_mmlu_full_yo_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_psychology.yaml
new file mode 100644
index 000000000..e1956c87c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_yo_social_sciences_tasks
+task: global_mmlu_full_yo_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_public_relations.yaml
new file mode 100644
index 000000000..5c6c2b8c1
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_yo_social_sciences_tasks
+task: global_mmlu_full_yo_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_security_studies.yaml
new file mode 100644
index 000000000..a12c4abd3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_yo_social_sciences_tasks
+task: global_mmlu_full_yo_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_sociology.yaml
new file mode 100644
index 000000000..e5747900b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_yo_social_sciences_tasks
+task: global_mmlu_full_yo_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_us_foreign_policy.yaml
new file mode 100644
index 000000000..493dda396
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_yo_social_sciences_tasks
+task: global_mmlu_full_yo_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_virology.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_virology.yaml
new file mode 100644
index 000000000..420b1b01c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_yo_other_tasks
+task: global_mmlu_full_yo_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_world_religions.yaml
new file mode 100644
index 000000000..c0964b307
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _yo_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_yo_humanities_tasks
+task: global_mmlu_full_yo_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/yo/utils.py b/lmms_eval/tasks/global_mmlu/full/yo/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/yo/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh.yaml b/lmms_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh.yaml
new file mode 100644
index 000000000..926c8f88e
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh.yaml
@@ -0,0 +1,11 @@
+group: global_mmlu_full_zh
+task:
+  - global_mmlu_full_zh_stem
+  - global_mmlu_full_zh_other
+  - global_mmlu_full_zh_social_sciences
+  - global_mmlu_full_zh_humanities
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_humanities.yaml b/lmms_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_humanities.yaml
new file mode 100644
index 000000000..fb347da8f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_humanities.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_zh_humanities
+task:
+  - global_mmlu_full_zh_humanities_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_other.yaml b/lmms_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_other.yaml
new file mode 100644
index 000000000..98d4ed5e6
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_other.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_zh_other
+task:
+  - global_mmlu_full_zh_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_social_sciences.yaml b/lmms_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_social_sciences.yaml
new file mode 100644
index 000000000..235012e6c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_social_sciences.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_zh_social_sciences
+task:
+  - global_mmlu_full_zh_social_sciences_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_stem.yaml b/lmms_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_stem.yaml
new file mode 100644
index 000000000..660486a4f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_stem.yaml
@@ -0,0 +1,8 @@
+group: global_mmlu_full_zh_stem
+task:
+  - global_mmlu_full_zh_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/_zh_template_yaml b/lmms_eval/tasks/global_mmlu/full/zh/_zh_template_yaml
new file mode 100644
index 000000000..2c83d4957
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/_zh_template_yaml
@@ -0,0 +1,16 @@
+dataset_path: CohereForAI/Global-MMLU
+dataset_name: zh
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_abstract_algebra.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_abstract_algebra.yaml
new file mode 100644
index 000000000..42ea6276b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_abstract_algebra.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_abstract_algebra
+tag: global_mmlu_full_zh_stem_tasks
+task: global_mmlu_full_zh_abstract_algebra
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_anatomy.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_anatomy.yaml
new file mode 100644
index 000000000..45001d14c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_anatomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_anatomy
+tag: global_mmlu_full_zh_stem_tasks
+task: global_mmlu_full_zh_anatomy
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_astronomy.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_astronomy.yaml
new file mode 100644
index 000000000..37183dc79
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_astronomy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_astronomy
+tag: global_mmlu_full_zh_stem_tasks
+task: global_mmlu_full_zh_astronomy
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_business_ethics.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_business_ethics.yaml
new file mode 100644
index 000000000..bbb5ea382
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_business_ethics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_business_ethics
+tag: global_mmlu_full_zh_other_tasks
+task: global_mmlu_full_zh_business_ethics
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_clinical_knowledge.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_clinical_knowledge.yaml
new file mode 100644
index 000000000..d90ee0ea4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_clinical_knowledge.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_clinical_knowledge
+tag: global_mmlu_full_zh_other_tasks
+task: global_mmlu_full_zh_clinical_knowledge
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_biology.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_biology.yaml
new file mode 100644
index 000000000..ba2031fef
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_college_biology
+tag: global_mmlu_full_zh_stem_tasks
+task: global_mmlu_full_zh_college_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_chemistry.yaml
new file mode 100644
index 000000000..860761b4f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_college_chemistry
+tag: global_mmlu_full_zh_stem_tasks
+task: global_mmlu_full_zh_college_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_computer_science.yaml
new file mode 100644
index 000000000..53d01965f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_college_computer_science
+tag: global_mmlu_full_zh_stem_tasks
+task: global_mmlu_full_zh_college_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_mathematics.yaml
new file mode 100644
index 000000000..dbd2e4be8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_college_mathematics
+tag: global_mmlu_full_zh_stem_tasks
+task: global_mmlu_full_zh_college_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_medicine.yaml
new file mode 100644
index 000000000..523d6b307
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_college_medicine
+tag: global_mmlu_full_zh_other_tasks
+task: global_mmlu_full_zh_college_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_physics.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_physics.yaml
new file mode 100644
index 000000000..0a08214f7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_college_physics
+tag: global_mmlu_full_zh_stem_tasks
+task: global_mmlu_full_zh_college_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_computer_security.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_computer_security.yaml
new file mode 100644
index 000000000..99332b356
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_computer_security.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_computer_security
+tag: global_mmlu_full_zh_stem_tasks
+task: global_mmlu_full_zh_computer_security
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_conceptual_physics.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_conceptual_physics.yaml
new file mode 100644
index 000000000..b042cc8bf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_conceptual_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_conceptual_physics
+tag: global_mmlu_full_zh_stem_tasks
+task: global_mmlu_full_zh_conceptual_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_econometrics.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_econometrics.yaml
new file mode 100644
index 000000000..bf9201121
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_econometrics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_econometrics
+tag: global_mmlu_full_zh_social_sciences_tasks
+task: global_mmlu_full_zh_econometrics
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_electrical_engineering.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_electrical_engineering.yaml
new file mode 100644
index 000000000..b30acad76
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_electrical_engineering.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_electrical_engineering
+tag: global_mmlu_full_zh_stem_tasks
+task: global_mmlu_full_zh_electrical_engineering
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_elementary_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_elementary_mathematics.yaml
new file mode 100644
index 000000000..3b108c421
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_elementary_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_elementary_mathematics
+tag: global_mmlu_full_zh_stem_tasks
+task: global_mmlu_full_zh_elementary_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_formal_logic.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_formal_logic.yaml
new file mode 100644
index 000000000..647755991
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_formal_logic.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_formal_logic
+tag: global_mmlu_full_zh_humanities_tasks
+task: global_mmlu_full_zh_formal_logic
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_global_facts.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_global_facts.yaml
new file mode 100644
index 000000000..07d390aab
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_global_facts.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_global_facts
+tag: global_mmlu_full_zh_other_tasks
+task: global_mmlu_full_zh_global_facts
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_biology.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_biology.yaml
new file mode 100644
index 000000000..28b2bdaa5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_biology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_high_school_biology
+tag: global_mmlu_full_zh_stem_tasks
+task: global_mmlu_full_zh_high_school_biology
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_chemistry.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_chemistry.yaml
new file mode 100644
index 000000000..4d084034d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_chemistry.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_high_school_chemistry
+tag: global_mmlu_full_zh_stem_tasks
+task: global_mmlu_full_zh_high_school_chemistry
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_computer_science.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_computer_science.yaml
new file mode 100644
index 000000000..6232ef607
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_computer_science.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_high_school_computer_science
+tag: global_mmlu_full_zh_stem_tasks
+task: global_mmlu_full_zh_high_school_computer_science
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_european_history.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_european_history.yaml
new file mode 100644
index 000000000..70e3e52bc
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_european_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_high_school_european_history
+tag: global_mmlu_full_zh_humanities_tasks
+task: global_mmlu_full_zh_high_school_european_history
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_geography.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_geography.yaml
new file mode 100644
index 000000000..fe6cb9139
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_geography.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_high_school_geography
+tag: global_mmlu_full_zh_social_sciences_tasks
+task: global_mmlu_full_zh_high_school_geography
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_government_and_politics.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_government_and_politics.yaml
new file mode 100644
index 000000000..cfa7213a4
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_government_and_politics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_high_school_government_and_politics
+tag: global_mmlu_full_zh_social_sciences_tasks
+task: global_mmlu_full_zh_high_school_government_and_politics
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_macroeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_macroeconomics.yaml
new file mode 100644
index 000000000..ca0b7ad86
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_macroeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_high_school_macroeconomics
+tag: global_mmlu_full_zh_social_sciences_tasks
+task: global_mmlu_full_zh_high_school_macroeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_mathematics.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_mathematics.yaml
new file mode 100644
index 000000000..38868e969
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_mathematics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_high_school_mathematics
+tag: global_mmlu_full_zh_stem_tasks
+task: global_mmlu_full_zh_high_school_mathematics
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_microeconomics.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_microeconomics.yaml
new file mode 100644
index 000000000..b79237d2f
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_microeconomics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_high_school_microeconomics
+tag: global_mmlu_full_zh_social_sciences_tasks
+task: global_mmlu_full_zh_high_school_microeconomics
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_physics.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_physics.yaml
new file mode 100644
index 000000000..6355da2f8
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_physics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_high_school_physics
+tag: global_mmlu_full_zh_stem_tasks
+task: global_mmlu_full_zh_high_school_physics
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_psychology.yaml
new file mode 100644
index 000000000..f22388673
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_high_school_psychology
+tag: global_mmlu_full_zh_social_sciences_tasks
+task: global_mmlu_full_zh_high_school_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_statistics.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_statistics.yaml
new file mode 100644
index 000000000..9aac20977
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_statistics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_high_school_statistics
+tag: global_mmlu_full_zh_stem_tasks
+task: global_mmlu_full_zh_high_school_statistics
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_us_history.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_us_history.yaml
new file mode 100644
index 000000000..47d8355f5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_us_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_high_school_us_history
+tag: global_mmlu_full_zh_humanities_tasks
+task: global_mmlu_full_zh_high_school_us_history
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_world_history.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_world_history.yaml
new file mode 100644
index 000000000..c1f6671f2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_world_history.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_high_school_world_history
+tag: global_mmlu_full_zh_humanities_tasks
+task: global_mmlu_full_zh_high_school_world_history
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_human_aging.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_human_aging.yaml
new file mode 100644
index 000000000..d6941ff7b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_human_aging.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_human_aging
+tag: global_mmlu_full_zh_other_tasks
+task: global_mmlu_full_zh_human_aging
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_human_sexuality.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_human_sexuality.yaml
new file mode 100644
index 000000000..ee228b22d
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_human_sexuality.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_human_sexuality
+tag: global_mmlu_full_zh_social_sciences_tasks
+task: global_mmlu_full_zh_human_sexuality
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_international_law.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_international_law.yaml
new file mode 100644
index 000000000..07b1ebd16
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_international_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_international_law
+tag: global_mmlu_full_zh_humanities_tasks
+task: global_mmlu_full_zh_international_law
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_jurisprudence.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_jurisprudence.yaml
new file mode 100644
index 000000000..ab10ffac5
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_jurisprudence.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_jurisprudence
+tag: global_mmlu_full_zh_humanities_tasks
+task: global_mmlu_full_zh_jurisprudence
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_logical_fallacies.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_logical_fallacies.yaml
new file mode 100644
index 000000000..451260b55
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_logical_fallacies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_logical_fallacies
+tag: global_mmlu_full_zh_humanities_tasks
+task: global_mmlu_full_zh_logical_fallacies
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_machine_learning.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_machine_learning.yaml
new file mode 100644
index 000000000..508d14f64
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_machine_learning.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_machine_learning
+tag: global_mmlu_full_zh_stem_tasks
+task: global_mmlu_full_zh_machine_learning
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_management.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_management.yaml
new file mode 100644
index 000000000..9db0b32bb
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_management.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_management
+tag: global_mmlu_full_zh_other_tasks
+task: global_mmlu_full_zh_management
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_marketing.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_marketing.yaml
new file mode 100644
index 000000000..a7142ce49
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_marketing.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_marketing
+tag: global_mmlu_full_zh_other_tasks
+task: global_mmlu_full_zh_marketing
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_medical_genetics.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_medical_genetics.yaml
new file mode 100644
index 000000000..220530907
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_medical_genetics.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_medical_genetics
+tag: global_mmlu_full_zh_other_tasks
+task: global_mmlu_full_zh_medical_genetics
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_miscellaneous.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_miscellaneous.yaml
new file mode 100644
index 000000000..5b479c9b0
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_miscellaneous.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_miscellaneous
+tag: global_mmlu_full_zh_other_tasks
+task: global_mmlu_full_zh_miscellaneous
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_moral_disputes.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_moral_disputes.yaml
new file mode 100644
index 000000000..58d13a994
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_moral_disputes.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_moral_disputes
+tag: global_mmlu_full_zh_humanities_tasks
+task: global_mmlu_full_zh_moral_disputes
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_moral_scenarios.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_moral_scenarios.yaml
new file mode 100644
index 000000000..95d91dfde
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_moral_scenarios.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_moral_scenarios
+tag: global_mmlu_full_zh_humanities_tasks
+task: global_mmlu_full_zh_moral_scenarios
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_nutrition.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_nutrition.yaml
new file mode 100644
index 000000000..57452a39b
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_nutrition.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_nutrition
+tag: global_mmlu_full_zh_other_tasks
+task: global_mmlu_full_zh_nutrition
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_philosophy.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_philosophy.yaml
new file mode 100644
index 000000000..20e237b24
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_philosophy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_philosophy
+tag: global_mmlu_full_zh_humanities_tasks
+task: global_mmlu_full_zh_philosophy
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_prehistory.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_prehistory.yaml
new file mode 100644
index 000000000..56358fe78
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_prehistory.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_prehistory
+tag: global_mmlu_full_zh_humanities_tasks
+task: global_mmlu_full_zh_prehistory
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_accounting.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_accounting.yaml
new file mode 100644
index 000000000..630681abf
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_accounting.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_professional_accounting
+tag: global_mmlu_full_zh_other_tasks
+task: global_mmlu_full_zh_professional_accounting
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_law.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_law.yaml
new file mode 100644
index 000000000..e48f35cb3
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_law.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_professional_law
+tag: global_mmlu_full_zh_humanities_tasks
+task: global_mmlu_full_zh_professional_law
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_medicine.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_medicine.yaml
new file mode 100644
index 000000000..f75432cd7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_medicine.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_professional_medicine
+tag: global_mmlu_full_zh_other_tasks
+task: global_mmlu_full_zh_professional_medicine
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_psychology.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_psychology.yaml
new file mode 100644
index 000000000..fbbf45ad7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_psychology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_professional_psychology
+tag: global_mmlu_full_zh_social_sciences_tasks
+task: global_mmlu_full_zh_professional_psychology
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_public_relations.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_public_relations.yaml
new file mode 100644
index 000000000..f760d2a29
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_public_relations.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_public_relations
+tag: global_mmlu_full_zh_social_sciences_tasks
+task: global_mmlu_full_zh_public_relations
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_security_studies.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_security_studies.yaml
new file mode 100644
index 000000000..1dafaf5d7
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_security_studies.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_security_studies
+tag: global_mmlu_full_zh_social_sciences_tasks
+task: global_mmlu_full_zh_security_studies
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_sociology.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_sociology.yaml
new file mode 100644
index 000000000..549f4ef15
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_sociology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_sociology
+tag: global_mmlu_full_zh_social_sciences_tasks
+task: global_mmlu_full_zh_sociology
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_us_foreign_policy.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_us_foreign_policy.yaml
new file mode 100644
index 000000000..597dcfa1c
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_us_foreign_policy.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_us_foreign_policy
+tag: global_mmlu_full_zh_social_sciences_tasks
+task: global_mmlu_full_zh_us_foreign_policy
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_virology.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_virology.yaml
new file mode 100644
index 000000000..1984c6b00
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_virology.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_virology
+tag: global_mmlu_full_zh_other_tasks
+task: global_mmlu_full_zh_virology
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_world_religions.yaml b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_world_religions.yaml
new file mode 100644
index 000000000..fa15c0cb2
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_world_religions.yaml
@@ -0,0 +1,5 @@
+# Generated by _generate_configs.py
+include: _zh_template_yaml
+process_docs: !function utils.process_world_religions
+tag: global_mmlu_full_zh_humanities_tasks
+task: global_mmlu_full_zh_world_religions
diff --git a/lmms_eval/tasks/global_mmlu/full/zh/utils.py b/lmms_eval/tasks/global_mmlu/full/zh/utils.py
new file mode 100644
index 000000000..7df72cb06
--- /dev/null
+++ b/lmms_eval/tasks/global_mmlu/full/zh/utils.py
@@ -0,0 +1,73 @@
+from functools import partial
+
+
+SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["subject"] == subject)
+
+
+process_functions = {
+    f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS
+}
+
+globals().update(process_functions)
diff --git a/lmms_eval/utils.py b/lmms_eval/utils.py
index e50b0aef9..65e1c1fc9 100755
--- a/lmms_eval/utils.py
+++ b/lmms_eval/utils.py
@@ -81,7 +81,7 @@ def escaped_split(text, sep_char, maxsplit=-1):
 
 ## final answer string exraction when using this parsing format
 def extract_final_answer(text):
-    match = re.search(r'Final Answer:\s*(.*)', text)
+    match = re.search(r'\nFinal Answer:\s*(.*)', text)
     if match:
         return match.group(1)
     else:
@@ -646,7 +646,8 @@ def load_yaml_config(yaml_path=None, yaml_config=None, yaml_dir=None, mode="full
 
     if yaml_dir is None:
         yaml_dir = os.path.dirname(yaml_path)
-
+    # breakpoint()
+    
     assert yaml_dir is not None
     assert yaml_config is not None
 

From e03a1b81f663c212248c70a6842d83ebd17e9a5a Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel.com>
Date: Mon, 19 May 2025 09:46:37 +0000
Subject: [PATCH 095/112] add llava_v6 model

---
 lmms_eval/models/__init__.py |  1 +
 lmms_eval/models/llava.py    | 20 ++++-----
 lmms_eval/models/llava_v6.py | 86 ++++++++++++++++++++++++++++++++++++
 3 files changed, 95 insertions(+), 12 deletions(-)
 create mode 100755 lmms_eval/models/llava_v6.py

diff --git a/lmms_eval/models/__init__.py b/lmms_eval/models/__init__.py
index 04185529a..c89095521 100644
--- a/lmms_eval/models/__init__.py
+++ b/lmms_eval/models/__init__.py
@@ -28,6 +28,7 @@
     "llama_vid": "LLaMAVid",
     "llama_vision": "LlamaVision",
     "llava": "Llava",
+    "llava_v6": "Llava_v6",
     "llava_hf": "LlavaHf",
     "llava_onevision": "Llava_OneVision",
     "llava_onevision_moviechat": "Llava_OneVision_MovieChat",
diff --git a/lmms_eval/models/llava.py b/lmms_eval/models/llava.py
index dd130ba2a..6be714402 100755
--- a/lmms_eval/models/llava.py
+++ b/lmms_eval/models/llava.py
@@ -314,7 +314,6 @@ def _collate(x):
         chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
         num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
         pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
-        breakpoint()
         for chunk in chunks:
             contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
             task = task[0]
@@ -367,19 +366,16 @@ def _collate(x):
                     question = image_tokens + "\n" + context
                 else:
                     question = context
-                # This is much safer for llama3, as we now have some object type in it
-                if "llama_3" in self.conv_template:
-                    conv = copy.deepcopy(conv_templates[self.conv_template])
+
+                if isinstance(question, list):
+                    conv = [ {"role": "user", "content": qq} for qq in question ],
                 else:
-                    conv = conv_templates[self.conv_template].copy()
-                # if conv.system:
-                #     conv.system = self.add_system_prompt
-                if self.add_system_prompt and conv.system:
-                    conv.system = self.add_system_prompt
-                conv.append_message(conv.roles[0], question)
-                conv.append_message(conv.roles[1], None)
-                prompt_question = conv.get_prompt()
+                    conv = [{"role": "user", "content": question}]
+                if self.add_system_prompt:
+                    conv.insert(0, {"role": "system", "content": self.add_system_prompt})
+                prompt_question = self._tokenizer.apply_chat_template(conv, tokenize=False, add_generation_prompt=True)
                 question_input.append(prompt_question)
+                
             
             # input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
             # preconfigure gen_kwargs with defaults
diff --git a/lmms_eval/models/llava_v6.py b/lmms_eval/models/llava_v6.py
new file mode 100755
index 000000000..44d176ad4
--- /dev/null
+++ b/lmms_eval/models/llava_v6.py
@@ -0,0 +1,86 @@
+from lmms_eval.models.llava import Llava
+import torch
+
+torch.backends.cuda.matmul.allow_tf32 = True
+
+
+import copy
+import warnings
+from datetime import timedelta
+from typing import List, Optional, Tuple, Union
+
+from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
+from accelerate.state import AcceleratorState
+from packaging import version
+from tqdm import tqdm
+
+from lmms_eval import utils
+from lmms_eval.api.instance import Instance
+from lmms_eval.api.model import lmms
+from lmms_eval.api.registry import register_model
+from lmms_eval.utils import stop_sequences_criteria
+
+warnings.filterwarnings("ignore")
+
+from loguru import logger as eval_logger
+
+try:
+    from llava.constants import DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX
+    from llava.conversation import conv_templates
+    from llava.mm_utils import (
+        get_model_name_from_path,
+        process_images,
+        tokenizer_image_token,
+    )
+    from llava.model.builder import load_pretrained_model
+except Exception as e:
+    eval_logger.debug("LLaVA is not installed. Please install LLaVA to use this model.\nError: %s" % e)
+
+# inference implementation for attention, can be "sdpa", "eager", "flash_attention_2". Seems FA2 is not effective during inference: https://discuss.huggingface.co/t/flash-attention-has-no-effect-on-inference/73453/5
+# if is_flash_attn_2_available:
+#     best_fit_attn_implementation = "flash_attention_2" # flash_attn has a bug that says: ERROR Error query and key must have the same dtype in generating
+
+if version.parse(torch.__version__) >= version.parse("2.1.2"):
+    best_fit_attn_implementation = "sdpa"
+else:
+    best_fit_attn_implementation = "eager"
+
+
+@register_model("llava_v6")
+class Llava_v6(Llava):
+    """
+    Llava Model
+    """
+
+    def __init__(
+        self,
+        pretrained: str = "liuhaotian/llava-v1.5-7b",
+        truncation: Optional[bool] = True,
+        device: Optional[str] = "cuda:0",
+        batch_size: Optional[Union[int, str]] = 1,
+        model_name=None,
+        attn_implementation=best_fit_attn_implementation,
+        device_map="cuda:0",
+        conv_template="qwen_2",
+        use_cache=True,
+        tie_weights: bool = True,
+        truncate_context=False,  # whether to truncate the context in generation, set it False for LLaVA-1.6
+        customized_config=None,  # ends in json
+        add_system_prompt=None,
+        **kwargs,
+    ) -> None:
+        super().__init__(
+            pretrained=pretrained,
+            truncation=truncation,
+            device=device,
+            batch_size=batch_size,
+            model_name=model_name,
+            attn_implementation=attn_implementation,
+            device_map=device_map,
+            conv_template=conv_template,
+            use_cache=use_cache,
+            tie_weights=tie_weights,
+            truncate_context=truncate_context,
+            customized_config=customized_config,
+            add_system_prompt=add_system_prompt,
+            **kwargs)
\ No newline at end of file

From 0e861540f156790905ba9d3ef0a8b2a182893f27 Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel.com>
Date: Mon, 19 May 2025 09:52:39 +0000
Subject: [PATCH 096/112] updated alm-bench pre-prompts

---
 lmms_eval/tasks/alm_bench/alm-bench-de.yaml | 9 ++++++---
 lmms_eval/tasks/alm_bench/alm-bench-en.yaml | 7 +++++--
 lmms_eval/tasks/alm_bench/alm-bench-es.yaml | 9 ++++++---
 lmms_eval/tasks/alm_bench/alm-bench-fr.yaml | 9 ++++++---
 lmms_eval/tasks/alm_bench/alm-bench-it.yaml | 9 ++++++---
 lmms_eval/tasks/alm_bench/alm-bench-ko.yaml | 9 ++++++---
 lmms_eval/tasks/alm_bench/alm-bench-nl.yaml | 9 ++++++---
 lmms_eval/tasks/alm_bench/alm-bench-pt.yaml | 9 ++++++---
 lmms_eval/tasks/alm_bench/alm-bench-ru.yaml | 9 ++++++---
 9 files changed, 53 insertions(+), 26 deletions(-)

diff --git a/lmms_eval/tasks/alm_bench/alm-bench-de.yaml b/lmms_eval/tasks/alm_bench/alm-bench-de.yaml
index 4d6ce6884..818014e9b 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-de.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-de.yaml
@@ -21,9 +21,12 @@ model_specific_target_kwargs:
 lmms_eval_specific_kwargs:
   default:
     prompt_format: "mcq"
-    # pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
-    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
\ No newline at end of file
+    post_prompt: " Answer:"
+  llava_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-en.yaml b/lmms_eval/tasks/alm_bench/alm-bench-en.yaml
index 0c3e4f424..5bd05c00f 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-en.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-en.yaml
@@ -21,9 +21,12 @@ model_specific_target_kwargs:
 lmms_eval_specific_kwargs:
   default:
     prompt_format: "mcq"
-    # pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
-    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
     post_prompt: " Answer:"
+  llava_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-es.yaml b/lmms_eval/tasks/alm_bench/alm-bench-es.yaml
index a91464679..4e3380beb 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-es.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-es.yaml
@@ -21,9 +21,12 @@ model_specific_target_kwargs:
 lmms_eval_specific_kwargs:
   default:
     prompt_format: "mcq"
-    # pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
-    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
\ No newline at end of file
+    post_prompt: " Answer:"
+  llava_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-fr.yaml b/lmms_eval/tasks/alm_bench/alm-bench-fr.yaml
index b3eff6fee..d6940e9d1 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-fr.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-fr.yaml
@@ -21,9 +21,12 @@ model_specific_target_kwargs:
 lmms_eval_specific_kwargs:
   default:
     prompt_format: "mcq"
-    # pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
-    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
\ No newline at end of file
+    post_prompt: " Answer:"
+  llava_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-it.yaml b/lmms_eval/tasks/alm_bench/alm-bench-it.yaml
index 912dd237c..9dc811e47 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-it.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-it.yaml
@@ -21,9 +21,12 @@ model_specific_target_kwargs:
 lmms_eval_specific_kwargs:
   default:
     prompt_format: "mcq"
-    # pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
-    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
\ No newline at end of file
+    post_prompt: " Answer:"
+  llava_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-ko.yaml b/lmms_eval/tasks/alm_bench/alm-bench-ko.yaml
index e3bd957a6..0ce940fc7 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-ko.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-ko.yaml
@@ -21,9 +21,12 @@ model_specific_target_kwargs:
 lmms_eval_specific_kwargs:
   default:
     prompt_format: "mcq"
-    # pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
-    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
\ No newline at end of file
+    post_prompt: " Answer:"
+  llava_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-nl.yaml b/lmms_eval/tasks/alm_bench/alm-bench-nl.yaml
index 4d8bf3c3f..b8f775340 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-nl.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-nl.yaml
@@ -21,9 +21,12 @@ model_specific_target_kwargs:
 lmms_eval_specific_kwargs:
   default:
     prompt_format: "mcq"
-    # pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
-    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
\ No newline at end of file
+    post_prompt: " Answer:"
+  llava_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-pt.yaml b/lmms_eval/tasks/alm_bench/alm-bench-pt.yaml
index 70623a2f7..8951e596f 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-pt.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-pt.yaml
@@ -21,9 +21,12 @@ model_specific_target_kwargs:
 lmms_eval_specific_kwargs:
   default:
     prompt_format: "mcq"
-    # pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
-    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
\ No newline at end of file
+    post_prompt: " Answer:"
+  llava_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-ru.yaml b/lmms_eval/tasks/alm_bench/alm-bench-ru.yaml
index 5adcf6ac2..a3d688a97 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-ru.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-ru.yaml
@@ -21,9 +21,12 @@ model_specific_target_kwargs:
 lmms_eval_specific_kwargs:
   default:
     prompt_format: "mcq"
-    # pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
-    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    pre_prompt: "For the given Multiple Choice Question Answer below, first analyze the question and then answer strictly from one of the options below."
     post_prompt: ""
   qwen_vl:
     pre_prompt: ""
-    post_prompt: " Answer:"
\ No newline at end of file
+    post_prompt: " Answer:"
+  llava_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
\ No newline at end of file

From 8810720a4b488d905acf4c124c846efbac20b989 Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel.com>
Date: Mon, 19 May 2025 09:53:23 +0000
Subject: [PATCH 097/112] minor fix in llava model

---
 lmms_eval/models/llava.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lmms_eval/models/llava.py b/lmms_eval/models/llava.py
index 6be714402..ba6217260 100755
--- a/lmms_eval/models/llava.py
+++ b/lmms_eval/models/llava.py
@@ -375,7 +375,6 @@ def _collate(x):
                     conv.insert(0, {"role": "system", "content": self.add_system_prompt})
                 prompt_question = self._tokenizer.apply_chat_template(conv, tokenize=False, add_generation_prompt=True)
                 question_input.append(prompt_question)
-                
             
             # input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
             # preconfigure gen_kwargs with defaults

From a0377ae7700e3f8139e1aebc60be6af72db3156b Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel.com>
Date: Tue, 20 May 2025 13:06:55 +0000
Subject: [PATCH 098/112] added the support of llava_v6 to the task

---
 lmms_eval/models/llava.py                   | 15 +++++++--------
 lmms_eval/tasks/ai2d/ai2d.yaml              |  6 +++++-
 lmms_eval/tasks/ai2d/utils.py               | 14 +++++++++-----
 lmms_eval/tasks/mmmu/_default_template_yaml |  2 +-
 lmms_eval/tasks/mmmu/mmmu.yaml              |  2 +-
 lmms_eval/tasks/mmmu/mmmu_val.yaml          |  4 ++++
 lmms_eval/tasks/mmmu/utils.py               |  4 +++-
 7 files changed, 30 insertions(+), 17 deletions(-)

diff --git a/lmms_eval/models/llava.py b/lmms_eval/models/llava.py
index ba6217260..896ecc156 100755
--- a/lmms_eval/models/llava.py
+++ b/lmms_eval/models/llava.py
@@ -1,5 +1,4 @@
 import torch
-
 torch.backends.cuda.matmul.allow_tf32 = True
 
 
@@ -96,11 +95,11 @@ def __init__(
         model_name = model_name if model_name is not None else get_model_name_from_path(pretrained)
         try:
             # Try to load the model with the multimodal argument
-            self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
+            self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, torch_dtype="bfloat16", **llava_model_args)
         except TypeError:
             # for older versions of LLaVA that don't have multimodal argument
             llava_model_args.pop("multimodal", None)
-            self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, **llava_model_args)
+            self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, torch_dtype="bfloat16", **llava_model_args)
         self._config = self._model.config
         self.model.eval()
         if tie_weights:
@@ -225,9 +224,9 @@ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
             if visuals:
                 image = process_images(visuals, self._image_processor, self._config)
                 if type(image) is list:
-                    image = [_image.to(dtype=torch.float16, device=self.device) for _image in image]
+                    image = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image]
                 else:
-                    image = image.to(dtype=torch.float16, device=self.device)
+                    image = image.to(dtype=torch.bfloat16, device=self.device)
             else:
                 image = None
 
@@ -314,6 +313,7 @@ def _collate(x):
         chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
         num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
         pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
+        # breakpoint()
         for chunk in chunks:
             contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
             task = task[0]
@@ -343,9 +343,9 @@ def _collate(x):
             if flattened_visuals:
                 image_tensor = process_images(flattened_visuals, self._image_processor, self._config)
                 if type(image_tensor) is list:
-                    image_tensor = [_image.to(dtype=torch.float16, device=self.device) for _image in image_tensor]
+                    image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
                 else:
-                    image_tensor = image_tensor.to(dtype=torch.float16, device=self.device)
+                    image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
             else:
                 image_tensor = None
 
@@ -432,7 +432,6 @@ def _collate(x):
             pbar.update(1)
             # reorder this group of results back to original unsorted form
         res = re_ords.get_original(res)
-
         pbar.close()
         return res
 
diff --git a/lmms_eval/tasks/ai2d/ai2d.yaml b/lmms_eval/tasks/ai2d/ai2d.yaml
index 3b4a33e02..baad8aa03 100755
--- a/lmms_eval/tasks/ai2d/ai2d.yaml
+++ b/lmms_eval/tasks/ai2d/ai2d.yaml
@@ -11,7 +11,7 @@ doc_to_target: !function utils.ai2d_doc_to_target
 lmms_eval_specific_kwargs:
   default:
     prompt_format: mcq
-    pre_prompt: ""
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below.\n"
     post_prompt: ""
     # post_prompt: "\nAnswer with the option's letter from the given choices directly."
   gpt4v: 
@@ -26,6 +26,10 @@ lmms_eval_specific_kwargs:
     prompt_format: mcq_xcomposer
     pre_prompt: "[UNUSED_TOKEN_146]user\nQuestion: "
     post_prompt: "[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\nThe answer is"
+  llava_v6: 
+    prompt_format: mcq
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below.\n"
+    post_prompt: "\nEnd your answer with 'Final Answer: <your answer>' where <your answer> is strictly the option letter from the given choices only."
 
 model_specific_target_kwargs:
   default: "mcq"
diff --git a/lmms_eval/tasks/ai2d/utils.py b/lmms_eval/tasks/ai2d/utils.py
index 3924a801b..8da40c9b2 100755
--- a/lmms_eval/tasks/ai2d/utils.py
+++ b/lmms_eval/tasks/ai2d/utils.py
@@ -2,7 +2,7 @@
 
 from lmms_eval.filters.extraction import ExtendedRegexFilter
 from lmms_eval.filters.transformation import MapFilter
-
+from lmms_eval.utils import extract_final_answer
 
 def ai2d_doc_to_text(doc, lmms_eval_specific_kwargs=None):
     question, choices = doc["question"], doc["options"]
@@ -57,22 +57,26 @@ def apply(self, resps, docs):
         # independently (and keep them a list.)
 
         filtered_resps = []
-
         for r, doc in zip(resps, docs):
             # Regex to directly extract the option letter from the model response
-            option_letter_regex = re.compile(r"^\s*([A-Z])\.")
+            """
+            # NOTE: Some models might only return the option letter (e.g., "A"), 
+            # while others might return the option letter followed by a full period (e.g., "A.").
+            """
+            option_letter_regex = re.compile(r"^\s*([A-Z])(\.)*")
 
             # Process each response
             filtered = []
             for resp in r:
                 # Try to match the option letter at the start of the response
-                match = option_letter_regex.match(resp)
+                pred = extract_final_answer(resp)
+                match = option_letter_regex.match(pred)
                 if match:
                     # If a match is found, append the matched letter
                     filtered.append(match.group(1))
                 else:
                     # If no match, return the original response
-                    filtered.append(resp)
+                    filtered.append(pred)
 
             # Assuming we need the first response that matches or the original response
             filtered_resps.append(filtered[0])
diff --git a/lmms_eval/tasks/mmmu/_default_template_yaml b/lmms_eval/tasks/mmmu/_default_template_yaml
index a53675347..1b1e202b4 100644
--- a/lmms_eval/tasks/mmmu/_default_template_yaml
+++ b/lmms_eval/tasks/mmmu/_default_template_yaml
@@ -1,5 +1,5 @@
 generation_kwargs:
-  max_new_tokens: 16
+  max_new_tokens: 1024
 
 metadata:
   version: 0.0
diff --git a/lmms_eval/tasks/mmmu/mmmu.yaml b/lmms_eval/tasks/mmmu/mmmu.yaml
index 935d03b76..970ef2090 100755
--- a/lmms_eval/tasks/mmmu/mmmu.yaml
+++ b/lmms_eval/tasks/mmmu/mmmu.yaml
@@ -1,4 +1,4 @@
 group: mmmu
 task:
 - mmmu_val
-- mmmu_test
+# - mmmu_test
diff --git a/lmms_eval/tasks/mmmu/mmmu_val.yaml b/lmms_eval/tasks/mmmu/mmmu_val.yaml
index 5d1394d25..f07c62971 100755
--- a/lmms_eval/tasks/mmmu/mmmu_val.yaml
+++ b/lmms_eval/tasks/mmmu/mmmu_val.yaml
@@ -18,5 +18,9 @@ lmms_eval_specific_kwargs:
     prompt_type: "format"
     multiple_choice_prompt: "Answer with the option's letter from the given choices directly."
     open_ended_prompt: "Answer the question using a single word or phrase."
+  llava_v6:
+    prompt_type: "format"
+    multiple_choice_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the option letter from the given choices only."
+    open_ended_prompt: "For the given Question, analyze the question and answer strictly using a single word or phrase. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the final answer (a single word or phrase) only."
 
 include: _default_template_yaml
\ No newline at end of file
diff --git a/lmms_eval/tasks/mmmu/utils.py b/lmms_eval/tasks/mmmu/utils.py
index 8a84a18cd..5abab710d 100755
--- a/lmms_eval/tasks/mmmu/utils.py
+++ b/lmms_eval/tasks/mmmu/utils.py
@@ -12,6 +12,7 @@
 import yaml
 from loguru import logger as eval_logger
 from openai import AzureOpenAI, OpenAI
+from lmms_eval.utils import extract_final_answer
 
 from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
 
@@ -394,6 +395,7 @@ def parse_multi_choice_response(response, all_choices, index2ans):
     Return the predicted index e.g., A, B, C, D.
     https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L10
     """
+    response = extract_final_answer(response)
     for char in [",", ".", "!", "?", ";", ":", "'"]:
         response = response.strip(char)
     response = " " + response + " "  # add space to avoid partial match
@@ -518,7 +520,6 @@ def parse_open_response(response):
     Return a list of predicted strings or numbers.
     https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L122
     """
-
     # content = content.strip("\n").strip(".").strip(" ")
     def get_key_subresponses(response):
         key_responses = []
@@ -567,6 +568,7 @@ def get_key_subresponses(response):
         return key_responses
 
     # pdb.set_trace()
+    response = extract_final_answer(response)
     key_responses = get_key_subresponses(response)
 
     pred_list = key_responses.copy()  # keep the original string response

From 612213f35ea40f3faca8e81750f2851cd19188ba Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel.com>
Date: Tue, 20 May 2025 13:11:53 +0000
Subject: [PATCH 099/112] small fix in the prompt

---
 lmms_eval/tasks/mmmu/mmmu_val.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lmms_eval/tasks/mmmu/mmmu_val.yaml b/lmms_eval/tasks/mmmu/mmmu_val.yaml
index f07c62971..34382f53c 100755
--- a/lmms_eval/tasks/mmmu/mmmu_val.yaml
+++ b/lmms_eval/tasks/mmmu/mmmu_val.yaml
@@ -21,6 +21,6 @@ lmms_eval_specific_kwargs:
   llava_v6:
     prompt_type: "format"
     multiple_choice_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the option letter from the given choices only."
-    open_ended_prompt: "For the given Question, analyze the question and answer strictly using a single word or phrase. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the final answer (a single word or phrase) only."
+    open_ended_prompt: "For the given question, analyze the question and answer strictly using a single word or phrase. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the final answer (a single word or phrase) only."
 
 include: _default_template_yaml
\ No newline at end of file

From d7394b0c27416dc2207d4a2af5ec66e1deb08916 Mon Sep 17 00:00:00 2001
From: aminfarajian <amin@unbabel.com>
Date: Wed, 21 May 2025 11:34:19 +0000
Subject: [PATCH 100/112] updated llava_hf and llava_v6

---
 lmms_eval/models/llava_hf.py |  24 ++++----
 lmms_eval/models/llava_v6.py | 108 +++++++++++++++--------------------
 2 files changed, 61 insertions(+), 71 deletions(-)

diff --git a/lmms_eval/models/llava_hf.py b/lmms_eval/models/llava_hf.py
index 03ab13c1a..3152767ef 100644
--- a/lmms_eval/models/llava_hf.py
+++ b/lmms_eval/models/llava_hf.py
@@ -66,7 +66,7 @@ def __init__(
         pretrained: str = "llava-hf/llava-1.5-7b-hf",
         revision: str = "main",
         device: str = "cuda",
-        dtype: Optional[Union[str, torch.dtype]] = "auto",
+        dtype: Optional[Union[str, torch.dtype]] = "bfloat16",
         batch_size: int = 1,
         trust_remote_code: Optional[bool] = False,
         attn_implementation: Optional[str] = None,
@@ -74,6 +74,7 @@ def __init__(
         chat_template: Optional[str] = None,
         use_cache: bool = True,
         max_frames_num: Optional[int] = 32,
+        add_system_prompt: Optional[str] = None,
         **kwargs,
     ) -> None:
         super().__init__()
@@ -136,7 +137,7 @@ def __init__(
             self._rank = 0
             self._world_size = 1
         self.accelerator = accelerator
-
+        self.add_system_prompt = add_system_prompt
     @property
     def config(self):
         # return the associated transformers.AutoConfig for the given pretrained model.
@@ -286,6 +287,7 @@ def _collate(x):
         chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
         num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
         pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
+        breakpoint()
         for chunk in chunks:
             contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
             task = task[0]
@@ -327,14 +329,16 @@ def _collate(x):
                 context = f"{image_tokens}\n{context}"
             # Apply chat template
             messages = [{"role": "user", "content": context}]
-            if self.chat_template is not None:
-                self.tokenizer.chat_template = self.chat_template
-                text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
-            elif self.tokenizer.chat_template is not None:
-                text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
-            else:
-                self.tokenizer.chat_template = VICUNA_CHAT_TEMPLATE
-                text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+            if self.add_system_prompt is not None:
+                messages.insert(0,{"role": "system", "content": self.add_system_prompt})
+            # if self.chat_template is not None:
+                # self.tokenizer.chat_template = self.chat_template
+            text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+            # elif self.tokenizer.chat_template is not None:
+                # text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+            # else:
+                # self.tokenizer.chat_template = VICUNA_CHAT_TEMPLATE
+                # text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
 
             if self.accelerator.is_main_process and doc_id[0] % 100 == 0:
                 eval_logger.debug(f"Prompt for doc ID {doc_id[0]}:\n\n{text}\n")
diff --git a/lmms_eval/models/llava_v6.py b/lmms_eval/models/llava_v6.py
index 44d176ad4..ed115d3fe 100755
--- a/lmms_eval/models/llava_v6.py
+++ b/lmms_eval/models/llava_v6.py
@@ -1,86 +1,72 @@
-from lmms_eval.models.llava import Llava
-import torch
-
-torch.backends.cuda.matmul.allow_tf32 = True
-
-
-import copy
+from lmms_eval.models.llava_hf import LlavaHf
 import warnings
-from datetime import timedelta
 from typing import List, Optional, Tuple, Union
 
-from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
-from accelerate.state import AcceleratorState
-from packaging import version
-from tqdm import tqdm
+import numpy as np
+import PIL
+import torch
+from transformers import (
+    LlavaForConditionalGeneration,
+    LlavaNextForConditionalGeneration,
+)
 
-from lmms_eval import utils
-from lmms_eval.api.instance import Instance
-from lmms_eval.api.model import lmms
 from lmms_eval.api.registry import register_model
-from lmms_eval.utils import stop_sequences_criteria
 
 warnings.filterwarnings("ignore")
 
 from loguru import logger as eval_logger
 
+DEFAULT_IMAGE_TOKEN = "<image>"
+DEFAULT_VIDEO_TOKEN = "<video>"
+
+model_map = {
+    "llava": LlavaForConditionalGeneration,
+    "llava_next": LlavaNextForConditionalGeneration,
+}
+
 try:
-    from llava.constants import DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX
-    from llava.conversation import conv_templates
-    from llava.mm_utils import (
-        get_model_name_from_path,
-        process_images,
-        tokenizer_image_token,
-    )
-    from llava.model.builder import load_pretrained_model
-except Exception as e:
-    eval_logger.debug("LLaVA is not installed. Please install LLaVA to use this model.\nError: %s" % e)
+    from transformers import LlavaOnevisionForConditionalGeneration
 
-# inference implementation for attention, can be "sdpa", "eager", "flash_attention_2". Seems FA2 is not effective during inference: https://discuss.huggingface.co/t/flash-attention-has-no-effect-on-inference/73453/5
-# if is_flash_attn_2_available:
-#     best_fit_attn_implementation = "flash_attention_2" # flash_attn has a bug that says: ERROR Error query and key must have the same dtype in generating
+    model_map["llava_onevision"] = LlavaOnevisionForConditionalGeneration
+except Exception as e:
+    eval_logger.debug("Transformers version does not support llava-onevision. Skipping.")
 
-if version.parse(torch.__version__) >= version.parse("2.1.2"):
-    best_fit_attn_implementation = "sdpa"
-else:
-    best_fit_attn_implementation = "eager"
 
 
 @register_model("llava_v6")
-class Llava_v6(Llava):
+class Llava_v6(LlavaHf):
     """
     Llava Model
     """
 
     def __init__(
         self,
-        pretrained: str = "liuhaotian/llava-v1.5-7b",
-        truncation: Optional[bool] = True,
-        device: Optional[str] = "cuda:0",
-        batch_size: Optional[Union[int, str]] = 1,
-        model_name=None,
-        attn_implementation=best_fit_attn_implementation,
-        device_map="cuda:0",
-        conv_template="qwen_2",
-        use_cache=True,
-        tie_weights: bool = True,
-        truncate_context=False,  # whether to truncate the context in generation, set it False for LLaVA-1.6
-        customized_config=None,  # ends in json
-        add_system_prompt=None,
+        pretrained: str = "llava-hf/llava-1.5-7b-hf",
+        revision: str = "main",
+        device: str = "cuda",
+        dtype: Optional[Union[str, torch.dtype]] = "bfloat16",
+        batch_size: int = 1,
+        trust_remote_code: Optional[bool] = False,
+        attn_implementation: Optional[str] = None,
+        device_map: str = "",
+        chat_template: Optional[str] = None,
+        use_cache: bool = True,
+        max_frames_num: Optional[int] = 32,
+        add_system_prompt: Optional[str] = None,
         **kwargs,
     ) -> None:
         super().__init__(
-            pretrained=pretrained,
-            truncation=truncation,
-            device=device,
-            batch_size=batch_size,
-            model_name=model_name,
-            attn_implementation=attn_implementation,
-            device_map=device_map,
-            conv_template=conv_template,
-            use_cache=use_cache,
-            tie_weights=tie_weights,
-            truncate_context=truncate_context,
-            customized_config=customized_config,
-            add_system_prompt=add_system_prompt,
-            **kwargs)
\ No newline at end of file
+            pretrained = pretrained,
+            revision = revision,
+            device = device,
+            dtype = dtype,
+            batch_size = batch_size,
+            trust_remote_code = trust_remote_code,
+            attn_implementation = attn_implementation,
+            device_map = device_map,
+            chat_template = chat_template,
+            use_cache = use_cache,
+            max_frames_num = max_frames_num,
+            add_system_prompt = add_system_prompt,
+            **kwargs
+            )
\ No newline at end of file

From d13ec67c40cdf15c0f34564988e6268c7c8148f6 Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Wed, 21 May 2025 14:59:45 +0000
Subject: [PATCH 101/112] temporary addition of m-wild-vision bench to switch
 to alm-bench branch

---
 .../m-wild-vision/m-wild-vision-gen-cs.yaml   | 29 +++++++
 .../m-wild-vision/m-wild-vision-gen-de.yaml   | 29 +++++++
 .../m-wild-vision/m-wild-vision-gen-eng.yaml  | 31 +++++++
 .../m-wild-vision/m-wild-vision-gen-es.yaml   | 29 +++++++
 .../m-wild-vision/m-wild-vision-gen-fr.yaml   | 29 +++++++
 .../m-wild-vision/m-wild-vision-gen-it.yaml   | 29 +++++++
 .../m-wild-vision/m-wild-vision-gen-ko.yaml   | 29 +++++++
 .../m-wild-vision/m-wild-vision-gen-nl.yaml   | 29 +++++++
 .../m-wild-vision/m-wild-vision-gen-pt.yaml   | 29 +++++++
 .../m-wild-vision/m-wild-vision-gen-ru.yaml   | 29 +++++++
 .../m-wild-vision/m-wild-vision-gen-zh.yaml   | 29 +++++++
 .../m-wild-vision/m-wild-vision-gen.yaml      | 14 ++++
 lmms_eval/tasks/m-wild-vision/utils.py        | 81 +++++++++++++++++++
 13 files changed, 416 insertions(+)
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-cs.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-de.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-eng.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-es.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-fr.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-it.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ko.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-nl.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-pt.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ru.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-zh.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/utils.py

diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-cs.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-cs.yaml
new file mode 100644
index 000000000..1a8b565a1
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-cs.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: ces_Latn
+task: m-wild-vision-gen-cs
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-de.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-de.yaml
new file mode 100644
index 000000000..4942525ea
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-de.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: deu_Latn
+task: m-wild-vision-gen-de
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-eng.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-eng.yaml
new file mode 100644
index 000000000..7c78d2c4c
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-eng.yaml
@@ -0,0 +1,31 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: eng_Latn
+task: m-wild-vision-gen-eng
+test_split: test
+output_type: generate_until
+
+process_docs: !function utils.process_docs
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-es.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-es.yaml
new file mode 100644
index 000000000..22c04fa8b
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-es.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: spa_Latn
+task: m-wild-vision-gen-es
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-fr.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-fr.yaml
new file mode 100644
index 000000000..6a2008b4a
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-fr.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: fra_Latn
+task: m-wild-vision-gen-fr
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-it.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-it.yaml
new file mode 100644
index 000000000..51511511b
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-it.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: ita_Latn
+task: m-wild-vision-gen-it
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ko.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ko.yaml
new file mode 100644
index 000000000..68503306c
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ko.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: kor_Hang
+task: m-wild-vision-gen-ko
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-nl.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-nl.yaml
new file mode 100644
index 000000000..20cfbc6cd
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-nl.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: nld_Latn
+task: m-wild-vision-gen-nl
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-pt.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-pt.yaml
new file mode 100644
index 000000000..3e08fa4c9
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-pt.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: por_Latn
+task: m-wild-vision-gen-pt
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ru.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ru.yaml
new file mode 100644
index 000000000..e389252d3
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ru.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: rus_Cyrl
+task: m-wild-vision-gen-ru
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-zh.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-zh.yaml
new file mode 100644
index 000000000..4d4ac9259
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-zh.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: zho_Hans
+task: m-wild-vision-gen-zh
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen.yaml
new file mode 100644
index 000000000..019330b74
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen.yaml
@@ -0,0 +1,14 @@
+group: m-wild-vision-gen
+task:
+  - m-wild-vision-gen-eng
+  - m-wild-vision-gen-de
+  - m-wild-vision-gen-es
+  - m-wild-vision-gen-fr
+  - m-wild-vision-gen-it
+  - m-wild-vision-gen-ko
+  - m-wild-vision-gen-zh
+  - m-wild-vision-gen-nl
+  - m-wild-vision-gen-pt
+  - m-wild-vision-gen-ru
+  # - m-wild-vision-gen-cs
+
diff --git a/lmms_eval/tasks/m-wild-vision/utils.py b/lmms_eval/tasks/m-wild-vision/utils.py
new file mode 100644
index 000000000..8b5d0ab6d
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/utils.py
@@ -0,0 +1,81 @@
+import base64
+from PIL import Image
+from io import BytesIO
+import numpy as np
+import re
+import json
+import os
+from loguru import logger
+
+
+def base64_to_bytes(base64_string):
+    # Remove the header if it exists (e.g., "data:image/jpeg;base64,")
+    if "base64," in base64_string:
+        base64_string = base64_string.split("base64,")[1]
+    # Decode base64 string to bytes
+    img_bytes = base64.b64decode(base64_string)
+    return img_bytes
+
+def process_docs(docs):
+    """
+    Process documents...
+    """
+    # logger.info(f"processing docs")
+    # Process images in place
+    docs = docs.select(range(5)) # filter out some samples!
+    return docs
+
+
+def gen_doc_to_visual(doc):
+    image = doc['image'][0]
+    image = image.convert('RGB')
+    return [image]
+
+
+def gen_doc_to_text(doc,lmms_eval_specific_kwargs=None ):
+    question = doc["instruction"]
+    if 'pre_prompt' in lmms_eval_specific_kwargs:
+        pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    else:
+        pre_prompt = ""
+    if 'post_prompt' in lmms_eval_specific_kwargs:
+        post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+    else:
+        post_prompt = ""
+    return f"{pre_prompt}Question: {question}\n{post_prompt}"
+
+
+# def doc_to_choice(doc,lmms_eval_specific_kwargs=None ):
+#     choices = doc["choices"]
+#     choice_list = [f"{label}: {text}" for label, text in zip(choices["label"], choices["text"])]
+#     choice_list = choices["text"]
+#     return choice_list
+
+# def doc_to_target(doc,lmms_eval_specific_kwargs=None ):
+#     choices = doc["choices"]
+#     answerKey = doc["answerKey"]
+#     import pdb; pdb.set_trace()
+#     return choices["label"].index(answerKey)
+
+def gen_process_results(doc, results):
+
+    generated_text = [res[0] for res in results]
+    
+    return {"results": {
+        "id": doc["question_id"],
+        "image": doc["image"],
+        "question": doc["instruction"],
+        "language": doc["language"],
+        "prediction": generated_text 
+        } 
+    }
+
+
+# def save_result_to_cache(doc, round_res, previous_round_info, save_dir):
+#     save_dict = dict(
+#         sample_id=doc["index"],
+#         question=doc["question"],
+#         round_res=round_res,
+#     )
+#     save_dict.update(previous_round_info)
+#     json.dump(save_dict, open(os.path.join(save_dir, f"{save_dict['sample_id']}.json"), "w"), indent=4)

From d293ce5d8a351ce389107081513517031a3e9925 Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Wed, 21 May 2025 15:01:19 +0000
Subject: [PATCH 102/112] updated gitignore with lm-eval-harness

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index e8552313b..3b58c84f8 100755
--- a/.gitignore
+++ b/.gitignore
@@ -42,3 +42,4 @@ VATEX/
 lmms_eval/tasks/vatex/__pycache__/utils.cpython-310.pyc
 lmms_eval/tasks/mlvu/__pycache__/utils.cpython-310.pyc
 .env
+lm-evaluation-harness/

From 0b2464cbc91c9b5d4f7ba71e65de316acff10779 Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Thu, 22 May 2025 17:04:50 +0000
Subject: [PATCH 103/112] removed breakpoint from llava_hf.py

---
 lmms_eval/models/llava_hf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lmms_eval/models/llava_hf.py b/lmms_eval/models/llava_hf.py
index 3152767ef..99cfb9fa5 100644
--- a/lmms_eval/models/llava_hf.py
+++ b/lmms_eval/models/llava_hf.py
@@ -287,7 +287,7 @@ def _collate(x):
         chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
         num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
         pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
-        breakpoint()
+
         for chunk in chunks:
             contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
             task = task[0]

From 6659b9cc0ed5cd62e7ae07e9343df22d30d2d346 Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Thu, 22 May 2025 18:30:38 +0000
Subject: [PATCH 104/112] ayavision and m-wild bench

---
 .../aya-vision-bench-gen-cs.yaml              |  34 +++
 .../aya-vision-bench-gen-de.yaml              |  34 +++
 .../aya-vision-bench-gen-eng.yaml             |  31 +++
 .../aya-vision-bench-gen-es.yaml              |  34 +++
 .../aya-vision-bench-gen-fr.yaml              |  35 +++
 .../aya-vision-bench-gen-it.yaml              |  35 +++
 .../aya-vision-bench-gen-ko.yaml              |  35 +++
 .../aya-vision-bench-gen-nl.yaml              |  35 +++
 .../aya-vision-bench-gen-pt.yaml              |  35 +++
 .../aya-vision-bench-gen-ru.yaml              |  35 +++
 .../aya-vision-bench-gen-zh.yaml              |  35 +++
 .../ayavisionbench/aya-vision-bench-gen.yaml  |  12 +
 lmms_eval/tasks/ayavisionbench/judge_eval.py  | 151 +++++++++++
 .../tasks/ayavisionbench/judge_templates.py   |  39 +++
 lmms_eval/tasks/ayavisionbench/judge_utils.py | 255 ++++++++++++++++++
 lmms_eval/tasks/ayavisionbench/utils.py       | 131 +++++++++
 .../m-wild-vision/m-wild-vision-gen-cs.yaml   |  29 ++
 .../m-wild-vision/m-wild-vision-gen-de.yaml   |  29 ++
 .../m-wild-vision/m-wild-vision-gen-eng.yaml  |  31 +++
 .../m-wild-vision/m-wild-vision-gen-es.yaml   |  29 ++
 .../m-wild-vision/m-wild-vision-gen-fr.yaml   |  29 ++
 .../m-wild-vision/m-wild-vision-gen-it.yaml   |  29 ++
 .../m-wild-vision/m-wild-vision-gen-ko.yaml   |  29 ++
 .../m-wild-vision/m-wild-vision-gen-nl.yaml   |  29 ++
 .../m-wild-vision/m-wild-vision-gen-pt.yaml   |  29 ++
 .../m-wild-vision/m-wild-vision-gen-ru.yaml   |  29 ++
 .../m-wild-vision/m-wild-vision-gen-zh.yaml   |  29 ++
 .../m-wild-vision/m-wild-vision-gen.yaml      |  12 +
 lmms_eval/tasks/m-wild-vision/utils.py        |  86 ++++++
 29 files changed, 1385 insertions(+)
 create mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml
 create mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml
 create mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-eng.yaml
 create mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml
 create mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml
 create mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml
 create mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml
 create mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml
 create mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml
 create mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml
 create mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml
 create mode 100644 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen.yaml
 create mode 100644 lmms_eval/tasks/ayavisionbench/judge_eval.py
 create mode 100644 lmms_eval/tasks/ayavisionbench/judge_templates.py
 create mode 100644 lmms_eval/tasks/ayavisionbench/judge_utils.py
 create mode 100644 lmms_eval/tasks/ayavisionbench/utils.py
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-cs.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-de.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-eng.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-es.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-fr.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-it.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ko.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-nl.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-pt.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ru.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-zh.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen.yaml
 create mode 100644 lmms_eval/tasks/m-wild-vision/utils.py

diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml
new file mode 100644
index 000000000..d35aca642
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml
@@ -0,0 +1,34 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: ces_Latn
+task: aya-vision-bench-gen-cs
+test_split: test
+output_type: generate_until
+process_docs: !function utils.process_docs
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+process_results: !function utils.gen_process_results
+
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml
new file mode 100644
index 000000000..043af92fb
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml
@@ -0,0 +1,34 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: deu_Latn
+task: aya-vision-bench-gen-de
+test_split: test
+output_type: generate_until
+process_docs: !function utils.process_docs
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+process_results: !function utils.gen_process_results
+
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-eng.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-eng.yaml
new file mode 100644
index 000000000..b9d0e7098
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-eng.yaml
@@ -0,0 +1,31 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: eng_Latn
+task: aya-vision-bench-gen-eng
+test_split: test
+output_type: generate_until
+
+process_docs: !function utils.process_docs
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+process_results: !function utils.gen_process_results
+
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0.0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Answer the given question."
+    post_prompt: "Answer:"
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml
new file mode 100644
index 000000000..df13935b1
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml
@@ -0,0 +1,34 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: spa_Latn
+task: aya-vision-bench-gen-es
+test_split: test
+output_type: generate_until
+
+process_docs: !function utils.process_docs
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+process_results: !function utils.gen_process_results
+
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml
new file mode 100644
index 000000000..3e624eef5
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml
@@ -0,0 +1,35 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: fra_Latn
+task: aya-vision-bench-gen-fr
+test_split: test
+output_type: generate_until
+
+process_docs: !function utils.process_docs
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+process_results: !function utils.gen_process_results
+
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml
new file mode 100644
index 000000000..110e4fc7b
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml
@@ -0,0 +1,35 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: ita_Latn
+task: aya-vision-bench-gen-it
+test_split: test
+output_type: generate_until
+
+process_docs: !function utils.process_docs
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+process_results: !function utils.gen_process_results
+
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml
new file mode 100644
index 000000000..12058e158
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml
@@ -0,0 +1,35 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: kor_Hang
+task: aya-vision-bench-gen-ko
+test_split: test
+output_type: generate_until
+
+process_docs: !function utils.process_docs
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+process_results: !function utils.gen_process_results
+
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml
new file mode 100644
index 000000000..c8b1f2c88
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml
@@ -0,0 +1,35 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: nld_Latn
+task: aya-vision-bench-gen-nl
+test_split: test
+output_type: generate_until
+
+process_docs: !function utils.process_docs
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+process_results: !function utils.gen_process_results
+
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml
new file mode 100644
index 000000000..96b8c8952
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml
@@ -0,0 +1,35 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: por_Latn
+task: aya-vision-bench-gen-pt
+test_split: test
+output_type: generate_until
+
+process_docs: !function utils.process_docs
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+process_results: !function utils.gen_process_results
+
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml
new file mode 100644
index 000000000..8eabc0b1b
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml
@@ -0,0 +1,35 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: rus_Cyrl
+task: aya-vision-bench-gen-ru
+test_split: test
+output_type: generate_until
+
+process_docs: !function utils.process_docs
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+process_results: !function utils.gen_process_results
+
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml
new file mode 100644
index 000000000..9c5093878
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml
@@ -0,0 +1,35 @@
+dataset_path: CohereForAI/AyaVisionBench
+dataset_name: zho_Hans
+task: aya-vision-bench-gen-zh
+test_split: test
+output_type: generate_until
+
+process_docs: !function utils.process_docs
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+process_results: !function utils.gen_process_results
+
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+# metric_list:
+#   - metric: results
+#     aggregation: !function utils.aggregate_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen.yaml
new file mode 100644
index 000000000..fed55c9e9
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen.yaml
@@ -0,0 +1,12 @@
+group: aya-vision-bench-gen
+task:
+  - aya-vision-bench-gen-eng
+  # - aya-vision-bench-gen-de
+  # - aya-vision-bench-gen-es
+  # - aya-vision-bench-gen-fr
+  # - aya-vision-bench-gen-it
+  # - aya-vision-bench-gen-ko
+  # - aya-vision-bench-gen-zh
+  # - aya-vision-bench-gen-nl
+  # - aya-vision-bench-gen-pt
+  # - aya-vision-bench-gen-ru
diff --git a/lmms_eval/tasks/ayavisionbench/judge_eval.py b/lmms_eval/tasks/ayavisionbench/judge_eval.py
new file mode 100644
index 000000000..1d19701f7
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/judge_eval.py
@@ -0,0 +1,151 @@
+import argparse
+import json
+from PIL import Image
+import os
+import io
+from loguru import logger
+
+from datasets import load_dataset
+from lmms_eval.tasks.ayavisionbench.judge_utils import run_judge,compute_results,parse_judge_responses
+
+def load_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--api_type",choices=["openai", "anthropic","litellm"], type=str, required=True)
+    parser.add_argument("--judge_model_name",default="gpt-4o-mini", type=str, required=True)
+    parser.add_argument("--judge_prompt_type",default="comparative",choices=["comparative", "direct assessment"], type=str, required=True)
+    parser.add_argument("--text_only",default=False, type=bool, required=True,help="if true, the judge model will not see the image.")
+    parser.add_argument("--model_output_path", type=str, required=True)
+    parser.add_argument("--baseline_output_path", type=str, required=True)
+    parser.add_argument("--lp", type=str, required=True)
+    parser.add_argument("--output_dir", type=str, required=True)
+    parser.add_argument("--save_judge_parsed_outputs",default=True, type=bool, required=True,help="if true, the judge parsed outputs will be saved.")
+    parser.add_argument("--save_judge_full_responses",default=True, type=bool, required=True,help="if true, the judge full responses will be saved.")
+    parser.add_argument("--save_judge_results",default=True, type=bool, required=True,help="if true, the judge results will be saved.")
+    parser.add_argument("--max_tokens",default=2048, type=int, required=True,help="the max tokens for the judge model.")
+    parser.add_argument("--temperature",default=0.0, type=float, required=True,help="the temperature for the judge model.")
+    parser.add_argument("--top_p",default=1.0, type=float, required=True,help="the top p for the judge model.")
+    parser.add_argument("--tensor_parallel_size",default=1, type=int, required=True,help="the tensor parallel size for the judge model.")
+    args = parser.parse_args()
+    return args
+
+def pil_to_image_dict(pil_img):
+    """
+    Convert a PIL Image to an image dict with 'bytes' and 'path' keys.
+    """
+    buf = io.BytesIO()
+    pil_img.save(buf, format="PNG")  # or "JPEG" if you prefer
+    img_bytes = buf.getvalue()
+    return {'bytes': img_bytes, 'path': None}
+
+def load_aya_vision_bench_data(language):
+    mapping_language_to_dataset_name = {
+        "cs": "ces_Latn",
+        "eng": "eng_Latn",
+        "fr": "fra_Latn",
+        "de": "deu_Latn",
+        "es": "spa_Latn",
+        "it": "ita_Latn",
+        "ko": "kor_Hang",
+        "nl": "nld_Latn",
+        "pt": "por_Latn",
+        "ru": "rus_Cyrl",
+        "zh": "zho_Hans",
+    }
+    dataset = load_dataset("CohereForAI/AyaVisionBench",name=mapping_language_to_dataset_name[language],split="test")
+    logger.info(f"Loaded Aya Vision Bench dataset for language: {language}")
+    # dataset = dataset.select(range(5))
+    # Extract questions and images from the dataset
+    questions = [item["question"] for item in dataset]
+
+    images_bytes = []
+    questions = []
+    for item in dataset:
+        questions.append(item["question"])
+        pil_img = item["image"][0]
+        images_bytes.append(pil_to_image_dict(pil_img))
+    return dataset,questions,images_bytes
+
+def load_model_outputs(model_outputs_path):
+    filtered_responses = []
+    with open(model_outputs_path, 'r', encoding='utf-8') as f:
+        for line in f:
+            try:
+                data = json.loads(line.strip())
+                if 'filtered_resps' in data:
+                    # Handle both single string and list of strings cases
+                    resps = data['filtered_resps']
+                    if isinstance(resps, list):
+                        filtered_responses.extend(resps)
+                    else:
+                        filtered_responses.append(resps)
+            except json.JSONDecodeError:
+                print(f"Warning: Skipping invalid JSON line")
+                continue
+    return filtered_responses
+
+def extract_judge_message(response):
+    """
+    Extracts the output message content from a judge response object.
+    Assumes response.choices[0].message.content exists.
+    """
+    try:
+        return response.choices[0].message.content
+    except (AttributeError, IndexError, KeyError, TypeError):
+        return None
+
+if __name__ == "__main__":
+    
+    args = load_args()
+
+    # 1. Load Aya Vision Bench data
+    aya_dataset,questions,images_bytes = load_aya_vision_bench_data(args.lp)
+
+    # 2. Load model outputs
+    baseline_model_outputs = load_model_outputs(args.baseline_output_path)
+    model_outputs = load_model_outputs(args.model_output_path)
+    logger.info(f"Loaded baseline and model outputs for language: {args.lp}")
+    
+    # 3. Prepare judge_config dictionary
+    judge_config = {
+        "api_type": args.api_type,
+        "judge_model_name": args.judge_model_name,
+        "judge_prompt_type": args.judge_prompt_type,
+        "text_only": args.text_only,
+        # "api_url": os.environ.get("API_URL", None),  # or set as needed
+        "max_tokens": args.max_tokens,
+        "temperature": args.temperature,
+        "top_p": args.top_p,
+        "tensor_parallel_size": args.tensor_parallel_size,
+        # Add any other config keys your run_judge expects
+    }
+
+    # 4. Run judge
+    logger.info(f"Running judge for language: {args.lp}")
+    full_responses = run_judge(questions, model_outputs, judge_config, baseline_model_outputs, images_bytes)
+    f_resp = [extract_judge_message(resp) for resp in full_responses]
+
+    logger.info(f"Judge completed for language: {args.lp}")
+    parsed_responses = parse_judge_responses(full_responses, judge_config)
+    logger.info(f"Parsed responses completed for language: {args.lp}")
+    
+    # 7. Compute results
+    results = compute_results(parsed_responses, judge_config)
+    breakpoint()
+    # 5. Save results
+    if args.save_judge_parsed_outputs:
+        with open(os.path.join(args.output_dir, "judge_results_parsed.json"), "w") as f:
+            logger.info(f"Saving parsed responses...")
+            json.dump(parsed_responses, f)
+
+    if args.save_judge_full_responses:
+        with open(os.path.join(args.output_dir, "judge_results_full.json"), "w") as f:
+            logger.info(f"Saving full responses...")
+            json.dump(f_resp, f)
+
+    if args.save_judge_results:
+        with open(os.path.join(args.output_dir, "judge_results.json"), "w") as f:
+            logger.info(f"Saving results...")
+            json.dump(results, f)
+
+    
+    
\ No newline at end of file
diff --git a/lmms_eval/tasks/ayavisionbench/judge_templates.py b/lmms_eval/tasks/ayavisionbench/judge_templates.py
new file mode 100644
index 000000000..6a635ed65
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/judge_templates.py
@@ -0,0 +1,39 @@
+COMPARATIVE_GEN_SYSTEM_PROMPT = """\
+Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user prompt displayed below. You will be given assistant A's answer and assistant B's answer. Your job is to evaluate which assistant's answer is better, by taking into account the given image.
+
+Begin your evaluation by generating your own answer to the prompt. You must provide your answers before judging any answers. To provide your answer, take into account the given image.
+
+When evaluating the assistants' answers, compare both assistants' answers with your answer. You must identify and correct any mistakes or inaccurate information.
+
+Then consider if the assistant's answers are helpful, relevant, and concise. Helpful means the answer correctly responds to the prompt or follows the instructions. Note when user prompt has any ambiguity or more than one interpretation, it is more helpful and appropriate to ask for clarifications or more information from the user than providing an answer based on assumptions. Relevant means all parts of the response closely connect or are appropriate to what is being asked. Concise means the response is clear and not verbose or excessive.
+
+Then consider the creativity and novelty of the assistant's answers when needed. Finally, identify any missing important information in the assistants' answers that would be beneficial to include when responding to the user prompt.
+
+After providing your explanation, you must output only one of the following choices as your final verdict with a label:
+
+1. Assistant A is significantly better: [[A>>B]]
+2. Assistant A is slightly better: [[A>B]]
+3. Tie, relatively the same: [[A=B]]
+4. Assistant B is slightly better: [[B>A]]
+5. Assistant B is significantly better: [[B>>A]]
+
+Example output: "My final verdict is tie: [[A=B]]".\
+"""
+
+COMPARATIVE_GEN_USER_PROMPT = """Question:{question}\n\n<|The Start of Assistant A's Answer|>\n{answer_1}\n<|The End of Assistant A's Answer|>\n\n<|The Start of Assistant B's Answer|>\n{answer_2}\n<|The End of Assistant B's Answer|>"""
+
+DIRECT_ASSESSMENT_SYSTEM_PROMPT = """\
+You are an impartial judge and will be given an image, a question, and an answer of an AI assistant. Your job is to evaluate the quality of the answer.
+
+Begin your evaluation by generating your own answer to the prompt. You must provide your answers before judging any answers.
+
+When evaluating the assistant's answer, compare the answer with your answer. You must identify and correct any mistakes or inaccurate information.
+
+Then consider if the assistant's answer is helpful, relevant, and concise. Helpful means the answer correctly responds to the prompt or follows the instructions. Note when user prompt has any ambiguity or more than one interpretation, it is more helpful and appropriate to ask for clarifications or more information from the user than providing an answer based on assumptions. Relevant means all parts of the response closely connect or are appropriate to what is being asked. Concise means the response is clear and not verbose or excessive.
+
+After providing your explanation, you must output only a score between 0 and 100, where 0 is the worst and 100 is the best.
+
+Example output: "My final score is 85."
+"""
+
+DIRECT_ASSESSMENT_USER_PROMPT = """Question:{question}\n\n<|The Start of Assistant Answer|>\n{answer}\n<|The End of Assistant Answer|>\n\n"""
\ No newline at end of file
diff --git a/lmms_eval/tasks/ayavisionbench/judge_utils.py b/lmms_eval/tasks/ayavisionbench/judge_utils.py
new file mode 100644
index 000000000..7b07d1567
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/judge_utils.py
@@ -0,0 +1,255 @@
+import yaml
+from pathlib import Path
+import os
+import requests
+import json
+import re
+from loguru import logger
+from dotenv import load_dotenv
+import base64
+import io
+from PIL import Image
+from tqdm import tqdm
+from typing import Dict, List, Optional, Union
+import litellm
+
+
+from lmms_eval.tasks.ayavisionbench.judge_templates import (
+    COMPARATIVE_GEN_USER_PROMPT,
+    COMPARATIVE_GEN_SYSTEM_PROMPT,
+    DIRECT_ASSESSMENT_SYSTEM_PROMPT,
+    DIRECT_ASSESSMENT_USER_PROMPT 
+)
+
+
+def img_bytes_to_url(image_dict: dict) -> str:
+    """
+    Convert image bytes to base64 URL.
+    
+    Args:
+        image_dict: Dictionary containing image bytes
+        
+    Returns:
+        str: base64 URL-encoded JPEG string
+    """
+    try:
+        # Convert bytes to PIL Image
+        image = Image.open(io.BytesIO(image_dict['bytes']))
+        
+        # Convert to RGB if needed
+        if image.mode != 'RGB':
+            image = image.convert('RGB')
+        
+        # Save as JPEG to buffer
+        buffer = io.BytesIO()
+        image.save(buffer, format="JPEG")
+        img_bytes = buffer.getvalue()
+        
+        # Convert to base64
+        base64_image = base64.b64encode(img_bytes).decode('utf-8')
+        
+        # Create data URL
+        data_url = f"data:image/jpeg;base64,{base64_image}"
+        
+        return data_url
+        
+    except Exception as e:
+        logger.error(f"Error converting image bytes to URL: {str(e)}")
+        return None
+
+# def get_judge_config():
+#     with open(Path(__file__).parent / "eval_judge_template.yaml", "r") as f:
+#         raw_data = f.readlines()
+#         safe_data = []
+#         for i, line in enumerate(raw_data):
+#             # remove function definition since yaml load cannot handle it
+#             if "!function" not in line:
+#                 safe_data.append(line)
+#         config = yaml.safe_load("".join(safe_data))
+#     return config
+
+
+def set_prompts(judge_config,questions,preds,baseline_model_outputs=None):
+    if judge_config["judge_prompt_type"] == "comparative":
+        system_prompt = COMPARATIVE_GEN_SYSTEM_PROMPT
+        user_prompt_template = COMPARATIVE_GEN_USER_PROMPT
+        prompts = [user_prompt_template.format(question=question,answer_1=base_output,answer_2=pred) for question,pred,base_output in zip(questions,preds,baseline_model_outputs)]
+    elif judge_config["judge_prompt_type"] == "direct_assessment":
+        system_prompt = DIRECT_ASSESSMENT_SYSTEM_PROMPT
+        user_prompt_template = DIRECT_ASSESSMENT_USER_PROMPT
+        prompts = [user_prompt_template.format(question=question,answer=pred) for question,pred in zip(questions,preds)]
+    else:
+        raise ValueError(f"Invalid judge prompt type: {judge_config['judge_prompt_type']}")
+    return system_prompt, prompts
+
+
+
+def run_judge(
+    questions: List[str],
+    preds: List[str],
+    judge_config: Dict,
+    baseline_model_outputs: Optional[List[str]] = None,
+    images: Optional[Dict[str, Union[bytes, None]]] = None,
+) -> List[str]:
+    """
+    Run judge evaluation on predictions using LiteLLM.
+    
+    Args:
+        questions: List of questions to evaluate
+        preds: List of predicted outputs to evaluate
+        judge_config: Configuration dictionary for the judge
+        baseline_model_outputs: Optional list of baseline outputs to compare against
+        images: Optional dictionary containing image data with format {'bytes': bytes, 'path': None}
+        
+    Returns:
+        List[Dict]: List of parsed responses from the judge
+    """
+    logger.info(f"Selected judge type: {judge_config['judge_prompt_type']}")
+    load_dotenv()
+    
+    # Set up model configuration
+    model = judge_config["judge_model_name"]
+    
+
+
+    if judge_config["api_type"] == "anthropic":
+        if os.getenv("ANTHROPIC_API_KEY"):
+            api_key = os.getenv("ANTHROPIC_API_KEY")
+        else:
+            raise ValueError("No API key found. Please set ANTHROPIC_API_KEY environment variable or define the api_key in the judge_config")
+    elif judge_config["api_type"] == "openai":
+        if os.getenv("OPENAI_API_KEY"):
+            api_key = os.getenv("OPENAI_API_KEY")
+        else:
+            raise ValueError("No API key found. Please set OPENAI_API_KEY environment variable or define the api_key in the judge_config")
+    elif judge_config["api_type"] == "litellm":
+        if os.getenv("LITELLM_API_KEY"):
+            api_key = os.getenv("LITELLM_API_KEY")
+        else:
+            raise ValueError("No API key found. Please set LITELLM_API_KEY environment variable or define the api_key in the judge_config")
+    else:
+        raise ValueError("No API key found. Please set LITELLM_API_KEY, ANTHROPIC_API_KEY, or OPENAI_API_KEY environment variable or define the api_key in the judge_config")
+
+    
+
+    # if judge_config["text_only"] == False:
+    #     assert litellm.supports_vision(model=model),f"Selected judge model:{model} does not support vision"
+
+    # Get prompts
+    system_prompt, user_prompts = set_prompts(judge_config, questions, preds, baseline_model_outputs)
+    
+    responses = []
+    total_items = len(user_prompts)
+    logger.info(f"Running judge evaluation for {total_items} items...")
+    
+    with tqdm(total=total_items, 
+              desc="Running judge evaluation", 
+              bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]',
+              ncols=100) as pbar:
+              
+        for idx, (image, prompt) in enumerate(zip(images, user_prompts)):
+            try:
+                # Prepare messages based on whether image is present
+                if image is not None or judge_config["text_only"] == False:
+                    messages = [
+                        {"role": "system", "content": [{"type": "text", "text": system_prompt}]},
+                        {"role": "user", "content": [
+                            {"type": "text", "text": prompt},
+                            {"type": "image_url", "image_url": {"url": img_bytes_to_url(image)}}
+                        ]}
+                    ]
+                else:
+                    messages = [
+                        {"role": "system", "content": system_prompt},
+                        {"role": "user", "content": prompt}
+                    ]
+
+                # Make the LiteLLM call
+                response = litellm.completion(
+                    model=model,
+                    messages=messages,
+                    max_tokens= judge_config["max_tokens"] if "max_tokens" in judge_config else None,
+                    temperature=judge_config["temperature"] if "temperature" in judge_config else None,
+                    top_p=judge_config["top_p"] if "top_p" in judge_config else None,
+                    api_key=api_key,
+                    base_url=judge_config.get("api_url")
+                )
+                responses.append(response)
+                
+            except Exception as e:
+                logger.error(f"Error in LiteLLM call: {str(e)}")
+                responses.append(None)
+            
+            pbar.set_postfix({"Current": f"{idx+1}/{total_items}"})
+            pbar.update(1)
+    return responses
+
+def parse_comparative_response(response: str) -> str:
+    """
+    Extract verdict pattern from judge response.
+    Matches: [[A>>B]], [[A>B]], [[A=B]], [[B>A]], [[B>>A]]
+    
+    Args:
+        response: String containing judge's response
+        
+    Returns:
+        str: Matched verdict pattern or None if no match found
+    """
+    # Regex pattern to match all possible verdicts
+    pattern = r'\[\[(A>>B|A>B|A=B|B>A|B>>A)\]\]'
+    # Search for pattern in response
+    match = re.search(pattern, response)
+    # Return matched pattern or None
+    return match.group(0) if match else None
+
+
+def parse_judge_responses(responses,judge_config):
+    logger.info(f"Parsing responses for {judge_config['judge_prompt_type']} judge...")
+    parsed_responses = []
+    for response in responses:
+        if response is None:
+            parsed_responses.append(None)
+            continue
+        response_data = response["choices"][0]["message"]["content"]
+        if judge_config["judge_prompt_type"] == "comparative":
+            parsed_responses.append(parse_comparative_response(response_data))
+        elif judge_config["judge_prompt_type"] == "direct_assessment":
+            raise NotImplementedError("Direct assessment parsing not implemented yet!")
+    return parsed_responses
+
+
+def compute_results(responses,judge_config):
+    logger.info(f"Computing results for {judge_config['judge_prompt_type']} judge...")
+    if judge_config["judge_prompt_type"] == "comparative":
+        a_better_than_b = 0
+        a_significantly_better_than_b = 0
+        b_better_than_a = 0
+        b_significantly_better_than_a = 0
+        a_equal_to_b = 0
+        no_answer = 0
+        for response in responses:
+            if response == "[[A>>B]]":
+                a_significantly_better_than_b += 1  
+            elif response == "[[A>B]]":
+                a_better_than_b += 1
+            elif response == "[[B>>A]]":
+                b_significantly_better_than_a += 1
+            elif response == "[[B>A]]":
+                b_better_than_a += 1
+            elif response == "[[A=B]]":
+                a_equal_to_b += 1
+            else:
+                no_answer += 1
+        results = {
+            "baseline_better_than_model": a_better_than_b/len(responses),
+            "baseline_significantly_better_than_model": a_significantly_better_than_b/len(responses),
+            "model_better_than_baseline": b_better_than_a/len(responses),
+            "model_significantly_better_than_baseline": b_significantly_better_than_a/len(responses),
+            "model_equal_to_baseline": a_equal_to_b/len(responses),
+            "no_answer_matched": no_answer/len(responses)
+        }
+    elif judge_config["judge_prompt_type"] == "direct_assessment":
+        raise NotImplementedError("Direct assessment results computation not implemented yet!")
+    else:
+        raise ValueError(f"Invalid judge prompt type: {judge_config['judge_prompt_type']}")
+    return results
\ No newline at end of file
diff --git a/lmms_eval/tasks/ayavisionbench/utils.py b/lmms_eval/tasks/ayavisionbench/utils.py
new file mode 100644
index 000000000..6e4f2136a
--- /dev/null
+++ b/lmms_eval/tasks/ayavisionbench/utils.py
@@ -0,0 +1,131 @@
+from PIL import Image
+from io import BytesIO
+import numpy as np
+import json
+from loguru import logger
+from pathlib import Path
+import io
+from copy import deepcopy
+
+# from lmms_eval.tasks.ayavisionbench.judge_utils import (
+#     get_judge_config,
+#     run_judge,
+#     compute_results
+# )
+
+def pil_to_image_dict(pil_img):
+    """
+    Convert a PIL Image to an image dict with 'bytes' and 'path' keys.
+    """
+    buf = io.BytesIO()
+    pil_img.save(buf, format="PNG")  # or "JPEG" if you prefer
+    img_bytes = buf.getvalue()
+    return {'bytes': img_bytes, 'path': None}
+
+
+
+# def load_baseline_outputs(baseline_model_outputs_path):
+#     filtered_responses = []
+#     with open(baseline_model_outputs_path, 'r', encoding='utf-8') as f:
+#         for line in f:
+#             try:
+#                 data = json.loads(line.strip())
+#                 if 'filtered_resps' in data:
+#                     # Handle both single string and list of strings cases
+#                     resps = data['filtered_resps']
+#                     if isinstance(resps, list):
+#                         filtered_responses.extend(resps)
+#                     else:
+#                         filtered_responses.append(resps)
+#             except json.JSONDecodeError:
+#                 print(f"Warning: Skipping invalid JSON line")
+#                 continue
+#     return filtered_responses
+
+
+def process_docs(docs):
+    """
+    Process documents...
+    """
+    # logger.info(f"processing docs")
+    # docs = docs.select(range(2)) # filter out some samples!
+    def copy_image_fn(example):
+        example['copy_image'] = example['image']
+        return example
+    
+    docs = docs.map(copy_image_fn)
+    return docs
+
+
+def gen_doc_to_visual(doc):
+    image = doc['image'][0]
+    image = image.convert('RGB')
+    return [image]
+
+
+def gen_doc_to_text(doc,lmms_eval_specific_kwargs=None ):
+    question = doc["prompt"]
+    if 'pre_prompt' in lmms_eval_specific_kwargs:
+        pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    else:
+        pre_prompt = ""
+    if 'post_prompt' in lmms_eval_specific_kwargs:
+        post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+    else:
+        post_prompt = ""
+    return f"{pre_prompt}Question: {question}\n{post_prompt}"
+
+
+def gen_process_results(doc, results):
+    generated_texts = results[0]
+    myimg = doc['copy_image'][0]['bytes']
+    pil_img = Image.open(io.BytesIO(myimg))
+    return {"results": {
+        "id": doc["index"],
+        "image": pil_img,
+        "question": doc["prompt"],
+        "image_category": doc["image_source_category"],
+        "prediction": generated_texts
+        }
+    }
+
+# def aggregate_results(results):
+#     # preds = [result["prediction"] for result in results]
+#     # questions = [result["question"] for result in results]
+#     # images = [result["image"] for result in results]
+#     # # Assuming images is a list of PIL Images:
+#     # images_bytes = [pil_to_image_dict(img) for img in images.copy()]
+
+#     # judge_config = get_judge_config()
+
+#     # if judge_config["run_judge"]:
+#     #     logger.info("Judge is enabled, checking judge config...")
+#     #     if judge_config["judge_prompt_type"] == "comparative":
+#     #         logger.info("Judgement is set to comparative, checking baseline model outputs...")
+#     #         try:
+#     #             baseline_model_outputs = load_baseline_outputs(judge_config["baseline_model_outputs_path"])
+#     #             assert len(baseline_model_outputs) == len(preds)
+#     #             logger.info("Baseline model outputs loaded successfully.")
+#     #             judge_results = run_judge(questions,preds,judge_config,baseline_model_outputs,images_bytes)
+#     #         except Exception as e:
+#     #             logger.error(f"Failed while loading model outputs or running judge.")
+#     #             raise e
+            
+#     #     elif judge_config["judge_prompt_type"] == "direct assessment":
+#     #         preds = [result["prediction"] for result in results]
+#     #         judge_results = run_judge(questions,preds,judge_config,baseline_model_outputs=None,images=images_bytes)
+#     #     else:
+#     #         logger.error(f"Invalid judge prompt type: {judge_config['judge_prompt_type']}.Skipping judge...")
+#     #         judge_results = None
+        
+#     # else:
+#     #     judge_results = None
+#     #     logger.info("Judge is disabled, skipping judge...")
+
+#     # if judge_results is None:
+#     #     return {"judge_results": None}
+#     # else:
+#     #     results = compute_results(judge_results,judge_config)
+#     #     return results
+#     return {"judge_results": None}
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-cs.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-cs.yaml
new file mode 100644
index 000000000..1a8b565a1
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-cs.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: ces_Latn
+task: m-wild-vision-gen-cs
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-de.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-de.yaml
new file mode 100644
index 000000000..4942525ea
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-de.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: deu_Latn
+task: m-wild-vision-gen-de
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-eng.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-eng.yaml
new file mode 100644
index 000000000..7c78d2c4c
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-eng.yaml
@@ -0,0 +1,31 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: eng_Latn
+task: m-wild-vision-gen-eng
+test_split: test
+output_type: generate_until
+
+process_docs: !function utils.process_docs
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-es.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-es.yaml
new file mode 100644
index 000000000..22c04fa8b
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-es.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: spa_Latn
+task: m-wild-vision-gen-es
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-fr.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-fr.yaml
new file mode 100644
index 000000000..6a2008b4a
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-fr.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: fra_Latn
+task: m-wild-vision-gen-fr
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-it.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-it.yaml
new file mode 100644
index 000000000..51511511b
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-it.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: ita_Latn
+task: m-wild-vision-gen-it
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ko.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ko.yaml
new file mode 100644
index 000000000..68503306c
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ko.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: kor_Hang
+task: m-wild-vision-gen-ko
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-nl.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-nl.yaml
new file mode 100644
index 000000000..20cfbc6cd
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-nl.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: nld_Latn
+task: m-wild-vision-gen-nl
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-pt.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-pt.yaml
new file mode 100644
index 000000000..3e08fa4c9
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-pt.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: por_Latn
+task: m-wild-vision-gen-pt
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ru.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ru.yaml
new file mode 100644
index 000000000..e389252d3
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ru.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: rus_Cyrl
+task: m-wild-vision-gen-ru
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-zh.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-zh.yaml
new file mode 100644
index 000000000..4d4ac9259
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-zh.yaml
@@ -0,0 +1,29 @@
+dataset_path: CohereForAI/m-WildVision
+dataset_name: zho_Hans
+task: m-wild-vision-gen-zh
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.gen_doc_to_visual
+doc_to_text: !function utils.gen_doc_to_text
+doc_to_target: ""
+
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+process_results: !function utils.gen_process_results
+
+
+metadata:
+  - version: 0.0
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "Answer:"
+
+
+
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen.yaml
new file mode 100644
index 000000000..5d7255efe
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen.yaml
@@ -0,0 +1,12 @@
+group: m-wild-vision-gen
+task:
+  - m-wild-vision-gen-eng
+  # - m-wild-vision-gen-de
+  # - m-wild-vision-gen-es
+  # - m-wild-vision-gen-fr
+  # - m-wild-vision-gen-it
+  # - m-wild-vision-gen-ko
+  # - m-wild-vision-gen-zh
+  # - m-wild-vision-gen-nl
+  # - m-wild-vision-gen-pt
+  # - m-wild-vision-gen-ru
diff --git a/lmms_eval/tasks/m-wild-vision/utils.py b/lmms_eval/tasks/m-wild-vision/utils.py
new file mode 100644
index 000000000..755edfb94
--- /dev/null
+++ b/lmms_eval/tasks/m-wild-vision/utils.py
@@ -0,0 +1,86 @@
+import base64
+from PIL import Image
+from io import BytesIO
+import numpy as np
+import re
+import json
+import os
+from loguru import logger
+import io
+
+def base64_to_bytes(base64_string):
+    # Remove the header if it exists (e.g., "data:image/jpeg;base64,")
+    if "base64," in base64_string:
+        base64_string = base64_string.split("base64,")[1]
+    # Decode base64 string to bytes
+    img_bytes = base64.b64decode(base64_string)
+    return img_bytes
+
+
+def process_docs(docs):
+    """
+    Process documents...
+    """
+    # logger.info(f"processing docs")
+    # docs = docs.select(range(2)) # filter out some samples!
+    def copy_image_fn(example):
+        example['copy_image'] = example['image']
+        return example
+    
+    docs = docs.map(copy_image_fn)
+    return docs
+
+
+def gen_doc_to_visual(doc):
+    image = doc['image'][0]
+    image = image.convert('RGB')
+    return [image]
+
+
+def gen_doc_to_text(doc,lmms_eval_specific_kwargs=None ):
+    question = doc["instruction"]
+    if 'pre_prompt' in lmms_eval_specific_kwargs:
+        pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    else:
+        pre_prompt = ""
+    if 'post_prompt' in lmms_eval_specific_kwargs:
+        post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+    else:
+        post_prompt = ""
+    return f"{pre_prompt}Question: {question}\n{post_prompt}"
+
+
+# def doc_to_choice(doc,lmms_eval_specific_kwargs=None ):
+#     choices = doc["choices"]
+#     choice_list = [f"{label}: {text}" for label, text in zip(choices["label"], choices["text"])]
+#     choice_list = choices["text"]
+#     return choice_list
+
+# def doc_to_target(doc,lmms_eval_specific_kwargs=None ):
+#     choices = doc["choices"]
+#     answerKey = doc["answerKey"]
+#     import pdb; pdb.set_trace()
+#     return choices["label"].index(answerKey)
+
+def gen_process_results(doc, results):
+    generated_text = results[0]
+    myimg = doc['copy_image'][0]['bytes']
+    pil_img = Image.open(io.BytesIO(myimg))
+    return {"results": {
+        "id": doc["question_id"],
+        "image": pil_img,
+        "question": doc["instruction"],
+        "language": doc["language"],
+        "prediction": generated_text 
+        } 
+    }
+
+
+# def save_result_to_cache(doc, round_res, previous_round_info, save_dir):
+#     save_dict = dict(
+#         sample_id=doc["index"],
+#         question=doc["question"],
+#         round_res=round_res,
+#     )
+#     save_dict.update(previous_round_info)
+#     json.dump(save_dict, open(os.path.join(save_dir, f"{save_dict['sample_id']}.json"), "w"), indent=4)

From 918a58f9d78a616427050ffce9961a419cc087cc Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Thu, 22 May 2025 18:34:12 +0000
Subject: [PATCH 105/112] added support of all languages for ayavisionbench and
 m-wild-vision

---
 .../ayavisionbench/aya-vision-bench-gen.yaml   | 18 +++++++++---------
 .../tasks/m-wild-vision/m-wild-vision-gen.yaml | 18 +++++++++---------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen.yaml
index fed55c9e9..798262fef 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen.yaml
@@ -1,12 +1,12 @@
 group: aya-vision-bench-gen
 task:
   - aya-vision-bench-gen-eng
-  # - aya-vision-bench-gen-de
-  # - aya-vision-bench-gen-es
-  # - aya-vision-bench-gen-fr
-  # - aya-vision-bench-gen-it
-  # - aya-vision-bench-gen-ko
-  # - aya-vision-bench-gen-zh
-  # - aya-vision-bench-gen-nl
-  # - aya-vision-bench-gen-pt
-  # - aya-vision-bench-gen-ru
+  - aya-vision-bench-gen-de
+  - aya-vision-bench-gen-es
+  - aya-vision-bench-gen-fr
+  - aya-vision-bench-gen-it
+  - aya-vision-bench-gen-ko
+  - aya-vision-bench-gen-zh
+  - aya-vision-bench-gen-nl
+  - aya-vision-bench-gen-pt
+  - aya-vision-bench-gen-ru
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen.yaml
index 5d7255efe..5752b0e5c 100644
--- a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen.yaml
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen.yaml
@@ -1,12 +1,12 @@
 group: m-wild-vision-gen
 task:
   - m-wild-vision-gen-eng
-  # - m-wild-vision-gen-de
-  # - m-wild-vision-gen-es
-  # - m-wild-vision-gen-fr
-  # - m-wild-vision-gen-it
-  # - m-wild-vision-gen-ko
-  # - m-wild-vision-gen-zh
-  # - m-wild-vision-gen-nl
-  # - m-wild-vision-gen-pt
-  # - m-wild-vision-gen-ru
+  - m-wild-vision-gen-de
+  - m-wild-vision-gen-es
+  - m-wild-vision-gen-fr
+  - m-wild-vision-gen-it
+  - m-wild-vision-gen-ko
+  - m-wild-vision-gen-zh
+  - m-wild-vision-gen-nl
+  - m-wild-vision-gen-pt
+  - m-wild-vision-gen-ru

From f91294511b99b6d3a65f0021177d92e3b9e7453a Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Thu, 22 May 2025 18:38:03 +0000
Subject: [PATCH 106/112] added default pre-prompts and post-prompts in configs
 for ayavision and m-wild-vision

---
 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml | 2 +-
 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml | 2 +-
 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml | 2 +-
 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml | 2 +-
 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml | 2 +-
 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml | 2 +-
 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml | 2 +-
 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml | 2 +-
 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml | 2 +-
 lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml | 2 +-
 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-cs.yaml     | 2 +-
 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-de.yaml     | 2 +-
 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-eng.yaml    | 2 +-
 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-es.yaml     | 2 +-
 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-fr.yaml     | 2 +-
 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-it.yaml     | 2 +-
 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ko.yaml     | 2 +-
 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-nl.yaml     | 2 +-
 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-pt.yaml     | 2 +-
 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ru.yaml     | 2 +-
 lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-zh.yaml     | 2 +-
 21 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml
index d35aca642..46d263e29 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-cs.yaml
@@ -27,7 +27,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml
index 043af92fb..7cb217c92 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-de.yaml
@@ -27,7 +27,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml
index df13935b1..cb81c3ac6 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-es.yaml
@@ -27,7 +27,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml
index 3e624eef5..d26cabef8 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-fr.yaml
@@ -28,7 +28,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml
index 110e4fc7b..37e53ab94 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-it.yaml
@@ -28,7 +28,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml
index 12058e158..95951c062 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ko.yaml
@@ -28,7 +28,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml
index c8b1f2c88..0fac5129e 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-nl.yaml
@@ -28,7 +28,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml
index 96b8c8952..0e0ecb7c8 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-pt.yaml
@@ -28,7 +28,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml
index 8eabc0b1b..d7716c658 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-ru.yaml
@@ -28,7 +28,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 
diff --git a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml
index 9c5093878..c0d0c75de 100644
--- a/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml
+++ b/lmms_eval/tasks/ayavisionbench/aya-vision-bench-gen-zh.yaml
@@ -28,7 +28,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-cs.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-cs.yaml
index 1a8b565a1..48c2c8417 100644
--- a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-cs.yaml
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-cs.yaml
@@ -22,7 +22,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-de.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-de.yaml
index 4942525ea..ec80d64dd 100644
--- a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-de.yaml
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-de.yaml
@@ -22,7 +22,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-eng.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-eng.yaml
index 7c78d2c4c..2972e9558 100644
--- a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-eng.yaml
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-eng.yaml
@@ -24,7 +24,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-es.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-es.yaml
index 22c04fa8b..8537868b8 100644
--- a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-es.yaml
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-es.yaml
@@ -22,7 +22,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-fr.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-fr.yaml
index 6a2008b4a..7364eb48f 100644
--- a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-fr.yaml
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-fr.yaml
@@ -22,7 +22,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-it.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-it.yaml
index 51511511b..8dd464261 100644
--- a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-it.yaml
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-it.yaml
@@ -22,7 +22,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ko.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ko.yaml
index 68503306c..ce2c0c08d 100644
--- a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ko.yaml
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ko.yaml
@@ -22,7 +22,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-nl.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-nl.yaml
index 20cfbc6cd..e95fe8676 100644
--- a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-nl.yaml
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-nl.yaml
@@ -22,7 +22,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-pt.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-pt.yaml
index 3e08fa4c9..46e09040c 100644
--- a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-pt.yaml
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-pt.yaml
@@ -22,7 +22,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ru.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ru.yaml
index e389252d3..ec9249b38 100644
--- a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ru.yaml
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-ru.yaml
@@ -22,7 +22,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 
diff --git a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-zh.yaml b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-zh.yaml
index 4d4ac9259..82691a03b 100644
--- a/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-zh.yaml
+++ b/lmms_eval/tasks/m-wild-vision/m-wild-vision-gen-zh.yaml
@@ -22,7 +22,7 @@ metadata:
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: ""
+    pre_prompt: "Answer the given question."
     post_prompt: "Answer:"
 
 

From d2057f3b10720717e930ff4d1f6ac93f50c97d46 Mon Sep 17 00:00:00 2001
From: farajian <amin@unbabel.com>
Date: Wed, 4 Jun 2025 17:18:03 +0000
Subject: [PATCH 107/112] added the v6 prompts and models

---
 lmms_eval/models/__init__.py                |   3 +
 lmms_eval/models/aya_v6.py                  |  34 ++
 lmms_eval/models/llava.py                   |  28 +-
 lmms_eval/models/llava_next.py              | 439 ++++++++++++++++++++
 lmms_eval/models/pixtral_v6.py              |  67 +++
 lmms_eval/tasks/ai2d/ai2d.yaml              |  14 +-
 lmms_eval/tasks/alm_bench/alm-bench-de.yaml |  12 +
 lmms_eval/tasks/alm_bench/alm-bench-en.yaml |  12 +
 lmms_eval/tasks/alm_bench/alm-bench-es.yaml |  12 +
 lmms_eval/tasks/alm_bench/alm-bench-fr.yaml |  12 +
 lmms_eval/tasks/alm_bench/alm-bench-it.yaml |  12 +
 lmms_eval/tasks/alm_bench/alm-bench-ko.yaml |  12 +
 lmms_eval/tasks/alm_bench/alm-bench-nl.yaml |  12 +
 lmms_eval/tasks/alm_bench/alm-bench-pt.yaml |  12 +
 lmms_eval/tasks/alm_bench/alm-bench-ru.yaml |  12 +
 lmms_eval/tasks/mmmu/mmmu_val.yaml          |  13 +
 lmms_eval/utils.py                          |   2 +-
 17 files changed, 698 insertions(+), 10 deletions(-)
 create mode 100644 lmms_eval/models/aya_v6.py
 create mode 100755 lmms_eval/models/llava_next.py
 create mode 100644 lmms_eval/models/pixtral_v6.py

diff --git a/lmms_eval/models/__init__.py b/lmms_eval/models/__init__.py
index c89095521..29a40a8d6 100644
--- a/lmms_eval/models/__init__.py
+++ b/lmms_eval/models/__init__.py
@@ -28,6 +28,7 @@
     "llama_vid": "LLaMAVid",
     "llama_vision": "LlamaVision",
     "llava": "Llava",
+    "llava_next": "LlavaNext",
     "llava_v6": "Llava_v6",
     "llava_hf": "LlavaHf",
     "llava_onevision": "Llava_OneVision",
@@ -71,7 +72,9 @@
     "aria": "Aria",
     "pangea": "Pangea",
     "pixtral": "Pixtral",
+    "pixtral_v6": "Pixtral_v6",
     "aya": "Aya",
+    "aya_v6": "Aya_v6",
     "egogpt": "EgoGPT",
     "internvideo2_5": "InternVideo2_5",
     "videochat_flash": "VideoChat_Flash",
diff --git a/lmms_eval/models/aya_v6.py b/lmms_eval/models/aya_v6.py
new file mode 100644
index 000000000..744880c2e
--- /dev/null
+++ b/lmms_eval/models/aya_v6.py
@@ -0,0 +1,34 @@
+from lmms_eval.models.aya import Aya
+import warnings
+from typing import Optional, Union
+import torch
+
+from lmms_eval.api.model import lmms
+from lmms_eval.api.registry import register_model
+
+
+warnings.filterwarnings("ignore")
+
+DEFAULT_IMAGE_TOKEN = "<image>"
+
+@register_model("aya_v6")
+class Aya_v6(Aya):
+    def __init__(
+        self,
+        pretrained: str = "CohereForAI/aya-vision-8b",
+        device: str = "cuda",
+        dtype: Optional[Union[str, torch.dtype]] = "bfloat16",
+        batch_size: int = 1,
+        add_system_prompt: Optional[str] = None,
+        device_map: str = "",
+        use_cache: bool = True,
+        **kwargs,
+    ) -> None:
+        super().__init__(
+        pretrained = pretrained,
+        device = device,
+        dtype = dtype,
+        batch_size = batch_size,
+        add_system_prompt = add_system_prompt,
+        device_map = device_map,
+        use_cache = use_cache,)
\ No newline at end of file
diff --git a/lmms_eval/models/llava.py b/lmms_eval/models/llava.py
index 896ecc156..28040fa88 100755
--- a/lmms_eval/models/llava.py
+++ b/lmms_eval/models/llava.py
@@ -22,6 +22,13 @@
 
 from loguru import logger as eval_logger
 
+from transformers import (
+    AutoConfig,
+    AutoProcessor,
+    LlavaForConditionalGeneration,
+    LlavaNextForConditionalGeneration,
+)
+
 try:
     from llava.constants import DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX
     from llava.conversation import conv_templates
@@ -30,7 +37,6 @@
         process_images,
         tokenizer_image_token,
     )
-    from llava.model.builder import load_pretrained_model
 except Exception as e:
     eval_logger.debug("LLaVA is not installed. Please install LLaVA to use this model.\nError: %s" % e)
 
@@ -60,6 +66,9 @@ def __init__(
         attn_implementation=best_fit_attn_implementation,
         device_map="cuda:0",
         conv_template="qwen_2",
+        revision="main",
+        trust_remote_code: Optional[bool] = False,
+        dtype: Optional[Union[str, torch.dtype]] = "auto",
         use_cache=True,
         tie_weights: bool = True,
         truncate_context=False,  # whether to truncate the context in generation, set it False for LLaVA-1.6
@@ -92,14 +101,17 @@ def __init__(
             llava_model_args["attn_implementation"] = attn_implementation
         if "use_flash_attention_2" in kwargs:
             llava_model_args["use_flash_attention_2"] = kwargs["use_flash_attention_2"]
-        model_name = model_name if model_name is not None else get_model_name_from_path(pretrained)
+        
         try:
             # Try to load the model with the multimodal argument
-            self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, torch_dtype="bfloat16", **llava_model_args)
-        except TypeError:
-            # for older versions of LLaVA that don't have multimodal argument
-            llava_model_args.pop("multimodal", None)
-            self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, torch_dtype="bfloat16", **llava_model_args)
+            self._model = LlavaNextForConditionalGeneration.from_pretrained(pretrained, revision=revision, torch_dtype=dtype, device_map=self.device_map, trust_remote_code=trust_remote_code, attn_implementation=attn_implementation)
+            self.pretrained = pretrained
+            self._image_processor = AutoProcessor.from_pretrained(pretrained, revision=revision, trust_remote_code=trust_remote_code)
+            self._image_processor.tokenizer.padding_side = "left"
+            self._tokenizer = self._image_processor.tokenizer
+            self._config = self._model.config
+        except:
+            print("Error loading model with multimodal argument.")
         self._config = self._model.config
         self.model.eval()
         if tie_weights:
@@ -373,7 +385,7 @@ def _collate(x):
                     conv = [{"role": "user", "content": question}]
                 if self.add_system_prompt:
                     conv.insert(0, {"role": "system", "content": self.add_system_prompt})
-                prompt_question = self._tokenizer.apply_chat_template(conv, tokenize=False, add_generation_prompt=True)
+                prompt_question = self.tokenizer.apply_chat_template(conv, tokenize=False, add_generation_prompt=True)
                 question_input.append(prompt_question)
             
             # input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
diff --git a/lmms_eval/models/llava_next.py b/lmms_eval/models/llava_next.py
new file mode 100755
index 000000000..3e941d4a1
--- /dev/null
+++ b/lmms_eval/models/llava_next.py
@@ -0,0 +1,439 @@
+import torch
+torch.backends.cuda.matmul.allow_tf32 = True
+
+
+import copy
+import warnings
+from datetime import timedelta
+from typing import List, Optional, Tuple, Union
+
+from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
+from accelerate.state import AcceleratorState
+from packaging import version
+from tqdm import tqdm
+
+from lmms_eval import utils
+from lmms_eval.api.instance import Instance
+from lmms_eval.api.model import lmms
+from lmms_eval.api.registry import register_model
+from lmms_eval.utils import stop_sequences_criteria
+
+warnings.filterwarnings("ignore")
+
+from loguru import logger as eval_logger
+
+try:
+    from llava.constants import DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX
+    from llava.conversation import conv_templates
+    from llava.mm_utils import (
+        get_model_name_from_path,
+        process_images,
+        tokenizer_image_token,
+    )
+    from llava.model.builder import load_pretrained_model
+except Exception as e:
+    eval_logger.debug("LLaVA is not installed. Please install LLaVA to use this model.\nError: %s" % e)
+
+# inference implementation for attention, can be "sdpa", "eager", "flash_attention_2". Seems FA2 is not effective during inference: https://discuss.huggingface.co/t/flash-attention-has-no-effect-on-inference/73453/5
+# if is_flash_attn_2_available:
+#     best_fit_attn_implementation = "flash_attention_2" # flash_attn has a bug that says: ERROR Error query and key must have the same dtype in generating
+
+if version.parse(torch.__version__) >= version.parse("2.1.2"):
+    best_fit_attn_implementation = "sdpa"
+else:
+    best_fit_attn_implementation = "eager"
+
+
+@register_model("llava_next")
+class LlavaNext(lmms):
+    """
+    Llava Model
+    """
+
+    def __init__(
+        self,
+        pretrained: str = "liuhaotian/llava-v1.5-7b",
+        truncation: Optional[bool] = True,
+        device: Optional[str] = "cuda:0",
+        batch_size: Optional[Union[int, str]] = 1,
+        model_name=None,
+        attn_implementation=best_fit_attn_implementation,
+        device_map="cuda:0",
+        conv_template="qwen_2",
+        use_cache=True,
+        tie_weights: bool = True,
+        truncate_context=False,  # whether to truncate the context in generation, set it False for LLaVA-1.6
+        customized_config=None,  # ends in json
+        add_system_prompt=None,
+        **kwargs,
+    ) -> None:
+        super().__init__()
+        # Do not use kwargs for now
+        assert kwargs == {}, f"Unexpected kwargs: {kwargs}"
+        accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
+        accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs])
+        self.accelerator = accelerator
+        if accelerator.num_processes > 1:
+            self._device = torch.device(f"cuda:{accelerator.local_process_index}")
+            self.device_map = f"cuda:{accelerator.local_process_index}"
+        elif accelerator.num_processes == 1 and device_map == "auto":
+            self._device = torch.device(device)
+            self.device_map = device_map
+        else:
+            self._device = torch.device(f"cuda:{accelerator.local_process_index}")
+            self.device_map = f"cuda:{accelerator.local_process_index}"
+
+        llava_model_args = {
+            "multimodal": True,
+        }
+        if customized_config is not None:
+            llava_model_args["customized_config"] = customized_config
+        if attn_implementation is not None:
+            llava_model_args["attn_implementation"] = attn_implementation
+        if "use_flash_attention_2" in kwargs:
+            llava_model_args["use_flash_attention_2"] = kwargs["use_flash_attention_2"]
+        model_name = model_name if model_name is not None else get_model_name_from_path(pretrained)
+        try:
+            # Try to load the model with the multimodal argument
+            self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, torch_dtype="bfloat16", **llava_model_args)
+        except TypeError:
+            # for older versions of LLaVA that don't have multimodal argument
+            llava_model_args.pop("multimodal", None)
+            self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, None, model_name, device_map=self.device_map, torch_dtype="bfloat16", **llava_model_args)
+        self._config = self._model.config
+        self.model.eval()
+        if tie_weights:
+            self.model.tie_weights()
+
+        self.truncation = truncation
+        self.batch_size_per_gpu = int(batch_size)
+        self.conv_template = conv_template
+        self.use_cache = use_cache
+        self.truncate_context = truncate_context
+        # assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
+        if accelerator.num_processes > 1:
+            assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
+            # If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
+            # Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
+            # I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
+            if accelerator.distributed_type == DistributedType.DEEPSPEED:
+                kwargs = {
+                    "train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
+                    "train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
+                }
+                AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
+                eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
+
+            if accelerator.distributed_type == DistributedType.FSDP or accelerator.distributed_type == DistributedType.DEEPSPEED:
+                self._model = accelerator.prepare(self.model)
+            else:
+                self._model = accelerator.prepare_model(self.model, evaluation_mode=True)
+            self.accelerator = accelerator
+            if self.accelerator.is_local_main_process:
+                eval_logger.info(f"Using {accelerator.num_processes} devices with data parallelism")
+            self._rank = self.accelerator.local_process_index
+            self._world_size = self.accelerator.num_processes
+        elif accelerator.num_processes == 1 and device_map == "auto":
+            eval_logger.info(f"Using {accelerator.num_processes} devices with tensor parallelism")
+            self._rank = 0
+            self._world_size = 1
+        else:
+            eval_logger.info(f"Using single device: {self._device}")
+            self.model.to(self._device)
+            self._rank = 0
+            self._world_size = 1
+        self.add_system_prompt = add_system_prompt
+    @property
+    def config(self):
+        # return the associated transformers.AutoConfig for the given pretrained model.
+        return self._config
+
+    @property
+    def tokenizer(self):
+        return self._tokenizer
+
+    @property
+    def model(self):
+        # returns the model, unwrapping it if using Accelerate
+        if hasattr(self, "accelerator"):
+            return self.accelerator.unwrap_model(self._model)
+        else:
+            return self._model
+
+    @property
+    def eot_token_id(self):
+        # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
+        return self.tokenizer.eos_token_id
+
+    @property
+    def max_length(self):
+        return self._max_length
+
+    def pad_sequence(self, input_ids, batch_first, padding_value):
+        if self.tokenizer.padding_side == "left":
+            input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids]
+        input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value)
+        if self.tokenizer.padding_side == "left":
+            input_ids = torch.flip(input_ids, [1])
+        return input_ids
+
+    @property
+    def batch_size(self):
+        return self.batch_size_per_gpu
+
+    @property
+    def device(self):
+        return self._device
+
+    @property
+    def rank(self):
+        return self._rank
+
+    @property
+    def world_size(self):
+        return self._world_size
+
+    def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
+        """ """
+        add_special_tokens = False if add_special_tokens is None else add_special_tokens
+        encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
+        # left-truncate the encoded context to be at most `left_truncate_len` tokens long
+        if left_truncate_len:
+            encoding = encoding[-left_truncate_len:]
+        return encoding
+
+    def tok_decode(self, tokens):
+        try:
+            return self.tokenizer.decode(tokens)
+        except:
+            return self.tokenizer.decode([tokens])
+
+    def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
+        # TODO
+        res = []
+        pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
+        for contexts, doc_to_target, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
+            # encode, pad, and truncate contexts for this batch
+            if type(doc_to_target) == str:
+                continuation = doc_to_target
+            else:
+                continuation = doc_to_target(self.task_dict[task][split][doc_id])
+            visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
+            visuals = self.flatten(visuals)
+            image_sizes = [[visual.size[0], visual.size[1]] for visual in visuals]
+            if visuals:
+                image = process_images(visuals, self._image_processor, self._config)
+                if type(image) is list:
+                    image = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image]
+                else:
+                    image = image.to(dtype=torch.bfloat16, device=self.device)
+            else:
+                image = None
+
+            prompts_input = contexts[0] if isinstance(contexts, list) else contexts
+
+            if image is not None and len(image) != 0 and DEFAULT_IMAGE_TOKEN not in prompts_input:
+                """
+                Three senarios:
+                1. No image, and there for, no image token should be added.
+                2. image token is already specified in the context, so we don't need to add it.
+                3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
+                """
+                image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visuals)
+                image_tokens = " ".join(image_tokens)
+                prompts_input = image_tokens + "\n" + (contexts[0] if isinstance(contexts, list) else contexts)
+
+            # This is much safer for llama3, as we now have some object type in it
+            if "llama_3" in self.conv_template:
+                conv = copy.deepcopy(conv_templates[self.conv_template])
+            else:
+                conv = conv_templates[self.conv_template].copy()
+            if self.add_system_prompt and conv.system:
+                conv.system = self.add_system_prompt
+            conv.append_message(conv.roles[0], prompts_input)
+            conv.append_message(conv.roles[1], None)
+            prompt = conv.get_prompt()
+
+            pad_token_id = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
+            contxt_id = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
+            # Add the answer of the second role
+            # NOTE: We are stripping the answer here so that there is no leading space in the answer
+            conv.messages[1][1] = continuation.strip()
+
+            prompt = conv.get_prompt()
+            input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
+            labels = input_ids.clone()
+            # Context part no need to calculate for loss
+            labels[0, : contxt_id.shape[1]] = -100
+            
+            # NOTE: We mask the last two tokens in the labels, which are the EOS and the new line tokens.
+            # This is to avoid calculating loss on these tokens and have a more comparable loss to the other models.
+            labels[0, -2:] = -100
+            
+            with torch.inference_mode():
+                outputs = self.model(input_ids=input_ids, labels=labels, images=image, use_cache=True, image_sizes=image_sizes)
+            loss = outputs["loss"]
+            # loss = torch.exp(loss)
+            logits = outputs["logits"]
+            greedy_tokens = logits.argmax(dim=-1)
+            cont_toks = input_ids[:, contxt_id.shape[1] :]  # [1, seq]
+            greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : input_ids.shape[1]]  # [1, seq]
+            max_equal = (greedy_tokens == cont_toks).all()
+            res.append((float(loss.item()), bool(max_equal)))
+            pbar.update(1)
+        pbar.close()
+        return res
+
+    def flatten(self, input):
+        if not input or any(i is None for i in input):
+            return []
+        new_list = []
+        for i in input:
+            if i:
+                for j in i:
+                    new_list.append(j)
+        return new_list
+
+    def generate_until(self, requests: List[Instance]) -> List[str]:
+        res = []
+        def _collate(x):
+            # the negative sign on len(toks) sorts descending - this has a few advantages:
+            # - time estimates will always be over not underestimates, which is more useful for planning
+            # - to know the size of a batch when going through the list, you know the first one is always the batch
+            #   padded context length. this is useful to simplify the batching logic and more importantly to make
+            #   automatic adaptive batches much much easier to implement
+            # - any OOMs will happen right away rather than near the end
+            toks = self.tok_encode(x[0])
+            return -len(toks), x[0]
+
+        # we group requests by their generation_kwargs,
+        # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
+        # in the same batch.
+        re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
+        chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
+        num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
+        pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
+        # breakpoint()
+        for chunk in chunks:
+            contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
+            task = task[0]
+            split = split[0]
+            batched_visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]  # [B, N]
+            flattened_visuals = self.flatten(batched_visuals)  # [B*N]
+            # we assume all gen kwargs in the batch are the same
+            # this is safe to assume because the `grouper` object ensures it.
+            gen_kwargs = all_gen_kwargs[0]
+
+            # Set default values for until and max_new_tokens
+            until = [self.tok_decode(self.eot_token_id)]
+
+            # Update values from gen_kwargs if present
+            if "until" in gen_kwargs:
+                until = gen_kwargs.pop("until")
+                if isinstance(until, str):
+                    until = [until]
+                elif not isinstance(until, list):
+                    raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
+
+            if "image_aspect_ratio" in gen_kwargs.keys() and "image_aspect_ratio" not in self._config.__dict__:
+                # here we should pop it out of gen_kwargs so that it doesn't get passed to the model for next step of generation
+                self._config.image_aspect_ratio = gen_kwargs.pop("image_aspect_ratio")
+                eval_logger.info(f"Setting image aspect ratio: {self._config.image_aspect_ratio}")
+            # encode, pad, and truncate contexts for this batch
+            if flattened_visuals:
+                image_tensor = process_images(flattened_visuals, self._image_processor, self._config)
+                if type(image_tensor) is list:
+                    image_tensor = [_image.to(dtype=torch.bfloat16, device=self.device) for _image in image_tensor]
+                else:
+                    image_tensor = image_tensor.to(dtype=torch.bfloat16, device=self.device)
+            else:
+                image_tensor = None
+
+            # prompts_input = contexts[0]
+
+            question_input = []
+
+            for visual, context in zip(batched_visuals, contexts):
+                if image_tensor is not None and len(image_tensor) != 0 and DEFAULT_IMAGE_TOKEN not in context:
+                    """
+                    Three senarios:
+                    1. No image, and there for, no image token should be added.
+                    2. image token is already specified in the context, so we don't need to add it.
+                    3. image token is not specified in the context and there is image inputs, so we need to add it. In this case, we add the image token at the beginning of the context and add a new line.
+                    """
+                    image_tokens = [DEFAULT_IMAGE_TOKEN] * len(visual) if isinstance(visual, list) else [DEFAULT_IMAGE_TOKEN]
+                    image_tokens = " ".join(image_tokens)
+                    question = image_tokens + "\n" + context
+                else:
+                    question = context
+
+                if isinstance(question, list):
+                    conv = [ {"role": "user", "content": qq} for qq in question ],
+                else:
+                    conv = [{"role": "user", "content": question}]
+                if self.add_system_prompt:
+                    conv.insert(0, {"role": "system", "content": self.add_system_prompt})
+                prompt_question = self._tokenizer.apply_chat_template(conv, tokenize=False, add_generation_prompt=True)
+                question_input.append(prompt_question)
+            
+            # input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(self.device)
+            # preconfigure gen_kwargs with defaults
+            gen_kwargs["image_sizes"] = [flattened_visuals[idx].size for idx in range(len(flattened_visuals))]
+            if "max_new_tokens" not in gen_kwargs:
+                gen_kwargs["max_new_tokens"] = 1024
+            if "temperature" not in gen_kwargs:
+                gen_kwargs["temperature"] = 0
+            if "top_p" not in gen_kwargs:
+                gen_kwargs["top_p"] = None
+            if "num_beams" not in gen_kwargs:
+                gen_kwargs["num_beams"] = 1
+
+            input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
+            pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
+            input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
+            attention_masks = input_ids.ne(pad_token_ids).to(self.device)
+            # These steps are not in LLaVA's original code, but are necessary for generation to work
+            # TODO: attention to this major generation step...
+            try:
+                cont = self.model.generate(
+                    input_ids,
+                    attention_mask=attention_masks,
+                    pad_token_id=pad_token_ids,
+                    images=image_tensor,
+                    image_sizes=gen_kwargs["image_sizes"],
+                    do_sample=True if gen_kwargs["temperature"] > 0 else False,
+                    temperature=gen_kwargs["temperature"],
+                    top_p=gen_kwargs["top_p"],
+                    num_beams=gen_kwargs["num_beams"],
+                    max_new_tokens=gen_kwargs["max_new_tokens"],
+                    use_cache=self.use_cache,
+                )
+                text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)
+            except Exception as e:
+                raise e
+                eval_logger.error(f"Error {e} in generating")
+                cont = ""
+                text_outputs = [""]
+
+            # cont_toks_list = cont.tolist()
+            # for cont_toks, context in zip(cont_toks_list, contexts):
+            # discard context + left-padding toks if using causal decoder-only LMM
+            # if self.truncate_context:
+            #     cont_toks = cont_toks[input_ids.shape[1] :]
+            # use secondary stop seqs to cut off should-have-been-stopped content post-hoc
+            # if self.truncate_context:
+            #     for term in until:
+            #         if len(term) > 0:
+            #             # ignore '' separator,
+            #             # for seq2seq case where self.tok_decode(self.eot_token_id) = ''
+            #             text_outputs = text_outputs.split(term)[0]
+            res.extend(text_outputs)
+            self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
+            pbar.update(1)
+            # reorder this group of results back to original unsorted form
+        res = re_ords.get_original(res)
+        pbar.close()
+        return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation for LLaVA")
diff --git a/lmms_eval/models/pixtral_v6.py b/lmms_eval/models/pixtral_v6.py
new file mode 100644
index 000000000..915d6af55
--- /dev/null
+++ b/lmms_eval/models/pixtral_v6.py
@@ -0,0 +1,67 @@
+import warnings
+from typing import List, Optional, Tuple, Union
+import os
+import torch
+from accelerate import Accelerator, DistributedType
+from accelerate.state import AcceleratorState
+from tqdm import tqdm
+from lmms_eval.api.instance import Instance
+from lmms_eval.api.model import lmms
+from lmms_eval.api.registry import register_model
+from huggingface_hub import snapshot_download
+
+from vllm.sampling_params import SamplingParams
+
+import io
+import base64
+from vllm import LLM
+from vllm.sampling_params import SamplingParams
+from lmms_eval.models.pixtral import Pixtral
+
+warnings.filterwarnings("ignore")
+from loguru import logger as eval_logger
+
+DEFAULT_IMAGE_TOKEN = "<image>"
+
+@register_model("pixtral_v6")
+class Pixtral_v6(Pixtral):
+    """
+    Custom PIXTRAL model implementation using Hugging Face Transformers and remote code.
+
+    Example usage:
+
+    accelerate launch --num_processes=8 -m lmms_eval \
+        --model pixtral \
+        --model_args pretrained=mistralai/mistralai/Pixtral-12B-2409 \
+        --tasks mme \
+        --batch_size 1 \
+        --output_path ./logs/ \
+        --log_samples
+    """
+
+    def __init__(
+        self,
+        pretrained: str = "mistralai/Pixtral-12B-2409",
+        device: str = "cuda",
+        dtype: Optional[Union[str, torch.dtype]] = "bfloat16",
+        batch_size: int = 1,
+        add_system_prompt: Optional[str] = None,
+        tag: Optional[str] = None,
+        device_map: str = "",
+        use_cache: bool = True,
+        max_img_per_msg: int = 10,
+        tokenizer_mode: str = "mistral",
+        gpu_memory_utilization: float = 0.7,
+        **kwargs,
+    ) -> None:
+        super().__init__(pretrained=pretrained,
+                         device=device,
+                         dtype=dtype,
+                         batch_size=batch_size,
+                         add_system_prompt=add_system_prompt,
+                         tag=tag,
+                         device_map=device_map,
+                         use_cache=use_cache,
+                         max_img_per_msg=max_img_per_msg,
+                         tokenizer_mode=tokenizer_mode,
+                         gpu_memory_utilization=gpu_memory_utilization)
\ No newline at end of file
diff --git a/lmms_eval/tasks/ai2d/ai2d.yaml b/lmms_eval/tasks/ai2d/ai2d.yaml
index baad8aa03..f5188d509 100755
--- a/lmms_eval/tasks/ai2d/ai2d.yaml
+++ b/lmms_eval/tasks/ai2d/ai2d.yaml
@@ -30,13 +30,25 @@ lmms_eval_specific_kwargs:
     prompt_format: mcq
     pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below.\n"
     post_prompt: "\nEnd your answer with 'Final Answer: <your answer>' where <your answer> is strictly the option letter from the given choices only."
+  aya_v6: 
+    prompt_format: mcq
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below.\n"
+    post_prompt: "\nEnd your answer with 'Final Answer: <your answer>' where <your answer> is strictly the option letter from the given choices only."
+  llava_next: 
+    prompt_format: mcq
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below.\n"
+    post_prompt: "\nEnd your answer with 'Final Answer: <your answer>' where <your answer> is strictly the option letter from the given choices only."
+  pixtral_v6: 
+    prompt_format: mcq
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below.\n"
+    post_prompt: "\nEnd your answer with 'Final Answer: <your answer>' where <your answer> is strictly the option letter from the given choices only."
 
 model_specific_target_kwargs:
   default: "mcq"
   qwen_vl: "qa"
 
 generation_kwargs:
-  max_new_tokens: 8192
+  max_new_tokens: 2048
   temperature: 0
   do_sample: False
 
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-de.yaml b/lmms_eval/tasks/alm_bench/alm-bench-de.yaml
index 818014e9b..4da21198b 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-de.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-de.yaml
@@ -27,6 +27,18 @@ lmms_eval_specific_kwargs:
     pre_prompt: ""
     post_prompt: " Answer:"
   llava_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  aya_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  llava_next:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  pixtral_v6:
     prompt_format: "mcq"
     pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
     post_prompt: ""
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-en.yaml b/lmms_eval/tasks/alm_bench/alm-bench-en.yaml
index 5bd05c00f..18a219a3c 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-en.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-en.yaml
@@ -30,3 +30,15 @@ lmms_eval_specific_kwargs:
     prompt_format: "mcq"
     pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
     post_prompt: ""
+  aya_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  llava_next:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  pixtral_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-es.yaml b/lmms_eval/tasks/alm_bench/alm-bench-es.yaml
index 4e3380beb..e70536137 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-es.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-es.yaml
@@ -27,6 +27,18 @@ lmms_eval_specific_kwargs:
     pre_prompt: ""
     post_prompt: " Answer:"
   llava_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  aya_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  llava_next:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  pixtral_v6:
     prompt_format: "mcq"
     pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
     post_prompt: ""
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-fr.yaml b/lmms_eval/tasks/alm_bench/alm-bench-fr.yaml
index d6940e9d1..ba1cc3620 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-fr.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-fr.yaml
@@ -27,6 +27,18 @@ lmms_eval_specific_kwargs:
     pre_prompt: ""
     post_prompt: " Answer:"
   llava_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  aya_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  llava_next:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  pixtral_v6:
     prompt_format: "mcq"
     pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
     post_prompt: ""
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-it.yaml b/lmms_eval/tasks/alm_bench/alm-bench-it.yaml
index 9dc811e47..9b237e48c 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-it.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-it.yaml
@@ -27,6 +27,18 @@ lmms_eval_specific_kwargs:
     pre_prompt: ""
     post_prompt: " Answer:"
   llava_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  aya_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  llava_next:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  pixtral_v6:
     prompt_format: "mcq"
     pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
     post_prompt: ""
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-ko.yaml b/lmms_eval/tasks/alm_bench/alm-bench-ko.yaml
index 0ce940fc7..c56659cb2 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-ko.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-ko.yaml
@@ -27,6 +27,18 @@ lmms_eval_specific_kwargs:
     pre_prompt: ""
     post_prompt: " Answer:"
   llava_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  ayaa_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  llava_next:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  pixtral_v6:
     prompt_format: "mcq"
     pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
     post_prompt: ""
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-nl.yaml b/lmms_eval/tasks/alm_bench/alm-bench-nl.yaml
index b8f775340..198973015 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-nl.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-nl.yaml
@@ -27,6 +27,18 @@ lmms_eval_specific_kwargs:
     pre_prompt: ""
     post_prompt: " Answer:"
   llava_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  aya_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  llava_next:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  pixtral_v6:
     prompt_format: "mcq"
     pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
     post_prompt: ""
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-pt.yaml b/lmms_eval/tasks/alm_bench/alm-bench-pt.yaml
index 8951e596f..141f2d63b 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-pt.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-pt.yaml
@@ -27,6 +27,18 @@ lmms_eval_specific_kwargs:
     pre_prompt: ""
     post_prompt: " Answer:"
   llava_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  aya_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  llava_next:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  pixtral_v6:
     prompt_format: "mcq"
     pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
     post_prompt: ""
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-ru.yaml b/lmms_eval/tasks/alm_bench/alm-bench-ru.yaml
index a3d688a97..89bfcd369 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-ru.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-ru.yaml
@@ -27,6 +27,18 @@ lmms_eval_specific_kwargs:
     pre_prompt: ""
     post_prompt: " Answer:"
   llava_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  aya_v6:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  llava_next:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
+  pixtral_v6:
     prompt_format: "mcq"
     pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
     post_prompt: ""
\ No newline at end of file
diff --git a/lmms_eval/tasks/mmmu/mmmu_val.yaml b/lmms_eval/tasks/mmmu/mmmu_val.yaml
index 34382f53c..f6d843c33 100755
--- a/lmms_eval/tasks/mmmu/mmmu_val.yaml
+++ b/lmms_eval/tasks/mmmu/mmmu_val.yaml
@@ -22,5 +22,18 @@ lmms_eval_specific_kwargs:
     prompt_type: "format"
     multiple_choice_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the option letter from the given choices only."
     open_ended_prompt: "For the given question, analyze the question and answer strictly using a single word or phrase. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the final answer (a single word or phrase) only."
+  aya_v6:
+    prompt_type: "format"
+    multiple_choice_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the option letter from the given choices only."
+    open_ended_prompt: "For the given question, analyze the question and answer strictly using a single word or phrase. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the final answer (a single word or phrase) only."
+  llava_next:
+    prompt_type: "format"
+    multiple_choice_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the option letter from the given choices only."
+    open_ended_prompt: "For the given question, analyze the question and answer strictly using a single word or phrase. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the final answer (a single word or phrase) only."
+  pixtral_v6:
+    prompt_type: "format"
+    multiple_choice_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the option letter from the given choices only."
+    open_ended_prompt: "For the given question, analyze the question and answer strictly using a single word or phrase. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the final answer (a single word or phrase) only."
+
 
 include: _default_template_yaml
\ No newline at end of file
diff --git a/lmms_eval/utils.py b/lmms_eval/utils.py
index 65e1c1fc9..6c6940734 100755
--- a/lmms_eval/utils.py
+++ b/lmms_eval/utils.py
@@ -81,7 +81,7 @@ def escaped_split(text, sep_char, maxsplit=-1):
 
 ## final answer string exraction when using this parsing format
 def extract_final_answer(text):
-    match = re.search(r'\nFinal Answer:\s*(.*)', text)
+    match = re.search(r'\n*Final Answer:\s*(.*)', text)
     if match:
         return match.group(1)
     else:

From 29d0f33486106bec13f1d2752021c743a4238619 Mon Sep 17 00:00:00 2001
From: farajian <amin@unbabel.com>
Date: Mon, 9 Jun 2025 16:45:53 +0000
Subject: [PATCH 108/112] updates for the commute task

---
 lmms_eval/models/llava_hf.py                | 44 +++++-----
 lmms_eval/models/pixtral.py                 | 92 +++++++++++++++++++++
 lmms_eval/tasks/alm_bench/alm-bench-en.yaml |  6 +-
 lmms_eval/tasks/alm_bench/utils.py          |  1 +
 lmms_eval/tasks/commute/commute-ar.yaml     |  4 +-
 lmms_eval/tasks/commute/commute-cs.yaml     |  4 +-
 lmms_eval/tasks/commute/commute-de.yaml     |  4 +-
 lmms_eval/tasks/commute/commute-es.yaml     | 35 --------
 lmms_eval/tasks/commute/commute-fr.yaml     |  4 +-
 lmms_eval/tasks/commute/commute-ru.yaml     |  4 +-
 lmms_eval/tasks/commute/commute-zh.yaml     |  4 +-
 lmms_eval/tasks/commute/utils.py            |  6 +-
 12 files changed, 136 insertions(+), 72 deletions(-)
 delete mode 100644 lmms_eval/tasks/commute/commute-es.yaml

diff --git a/lmms_eval/models/llava_hf.py b/lmms_eval/models/llava_hf.py
index 99cfb9fa5..0dcc4698f 100644
--- a/lmms_eval/models/llava_hf.py
+++ b/lmms_eval/models/llava_hf.py
@@ -66,7 +66,7 @@ def __init__(
         pretrained: str = "llava-hf/llava-1.5-7b-hf",
         revision: str = "main",
         device: str = "cuda",
-        dtype: Optional[Union[str, torch.dtype]] = "bfloat16",
+        dtype: Optional[Union[str, torch.dtype]] = "auto",
         batch_size: int = 1,
         trust_remote_code: Optional[bool] = False,
         attn_implementation: Optional[str] = None,
@@ -74,7 +74,8 @@ def __init__(
         chat_template: Optional[str] = None,
         use_cache: bool = True,
         max_frames_num: Optional[int] = 32,
-        add_system_prompt: Optional[str] = None,
+        add_system_prompt: str = None,
+        add_bos_token: bool = False,
         **kwargs,
     ) -> None:
         super().__init__()
@@ -137,7 +138,7 @@ def __init__(
             self._rank = 0
             self._world_size = 1
         self.accelerator = accelerator
-        self.add_system_prompt = add_system_prompt
+
     @property
     def config(self):
         # return the associated transformers.AutoConfig for the given pretrained model.
@@ -223,13 +224,14 @@ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
                 self.tokenizer.chat_template = VICUNA_CHAT_TEMPLATE
                 prompt = self.tokenizer.apply_chat_template(messages[:-1], tokenize=False, add_generation_prompt=True)
                 prompt_and_continuation = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
-
             formatted_contexts = [prompt]
             formatted_continuation = [prompt_and_continuation]
             model_inputs = self._image_processor(text=formatted_continuation, images=visuals, return_tensors="pt").to(self._device, self.model.dtype)
             labels = model_inputs["input_ids"].clone()
             contxt_id = self._image_processor(text=formatted_contexts, return_tensors="pt")["input_ids"]
+            breakpoint()
             labels[:, : contxt_id.shape[1]] = -100
+            labels[0, -1:] = -100  # last token is also ignored
 
             if self.accelerator.is_main_process and doc_id % 100 == 0:
                 eval_logger.debug(f"Prompt for doc ID {doc_id}:\n\n{formatted_contexts[0]}\n")
@@ -287,7 +289,6 @@ def _collate(x):
         chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
         num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
         pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
-
         for chunk in chunks:
             contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
             task = task[0]
@@ -329,16 +330,14 @@ def _collate(x):
                 context = f"{image_tokens}\n{context}"
             # Apply chat template
             messages = [{"role": "user", "content": context}]
-            if self.add_system_prompt is not None:
-                messages.insert(0,{"role": "system", "content": self.add_system_prompt})
-            # if self.chat_template is not None:
-                # self.tokenizer.chat_template = self.chat_template
-            text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
-            # elif self.tokenizer.chat_template is not None:
-                # text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
-            # else:
-                # self.tokenizer.chat_template = VICUNA_CHAT_TEMPLATE
-                # text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+            if self.chat_template is not None:
+                self.tokenizer.chat_template = self.chat_template
+                text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+            elif self.tokenizer.chat_template is not None:
+                text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+            else:
+                self.tokenizer.chat_template = VICUNA_CHAT_TEMPLATE
+                text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
 
             if self.accelerator.is_main_process and doc_id[0] % 100 == 0:
                 eval_logger.debug(f"Prompt for doc ID {doc_id[0]}:\n\n{text}\n")
@@ -367,7 +366,8 @@ def _collate(x):
             if "num_beams" not in gen_kwargs:
                 gen_kwargs["num_beams"] = 1
             try:
-                cont = self.model.generate(
+                # breakpoint()
+                outputs = self.model.generate(
                     **inputs,
                     do_sample=True if gen_kwargs["temperature"] > 0 else False,
                     temperature=gen_kwargs["temperature"],
@@ -376,13 +376,13 @@ def _collate(x):
                     max_new_tokens=gen_kwargs["max_new_tokens"],
                     use_cache=self.use_cache,
                     pad_token_id=self.eot_token_id,
-                    eos_token_id=self.eot_token_id,
-                )
-                cont = cont[:, inputs["input_ids"].shape[-1] :]
+                    eos_token_id=self.eot_token_id)
+
+                outputs = outputs[:, inputs["input_ids"].shape[-1] :]
             except Exception as e:
                 eval_logger.error(f"Error {e} in generating")
-                cont = ""
-            text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)[0]
+                outputs = ""
+            text_outputs = self.tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]
             if self.accelerator.is_main_process and doc_id[0] % 100 == 0:
                 eval_logger.debug(f"Generated text for doc ID {doc_id[0]}:\n\n{text_outputs}\n")
 
@@ -396,4 +396,4 @@ def _collate(x):
         return res
 
     def generate_until_multi_round(self, requests) -> List[str]:
-        raise NotImplementedError("TODO: Implement multi-round generation for LLaVAHF")
+        raise NotImplementedError("TODO: Implement multi-round generation for LLaVAHF")
\ No newline at end of file
diff --git a/lmms_eval/models/pixtral.py b/lmms_eval/models/pixtral.py
index 1ba4f2dea..c6e60ef72 100644
--- a/lmms_eval/models/pixtral.py
+++ b/lmms_eval/models/pixtral.py
@@ -232,6 +232,98 @@ def get_image_url(self, image_bytes: bytes) -> str:
         data_url = f"data:image/jpeg;base64,{base64_image}"
         return data_url
 
+
+
+    def loglikelihood(self, requests: List[Instance]) -> List[str]:
+        res = []
+
+        def batch_requests(requests, batch_size):
+            for i in range(0, len(requests), batch_size):
+                yield [x.arguments for x in requests[i:i + batch_size]]
+
+        chunks = batch_requests(requests, self.batch_size)  
+        
+        num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
+        
+        pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
+        breakpoint()
+        for chunk in chunks:
+            contexts, doc_to_target, doc_to_visuals, doc_id, tasks, splits = zip(*chunk)
+            if type(doc_to_target) == str:
+                continuation = doc_to_target
+            
+            
+            visuals = [doc_to_visual(self.task_dict[task][split][ids]) for ids, task, split, doc_to_visual in zip(doc_id, tasks, splits, doc_to_visuals)]
+            
+            # Use the generation kwargs from the first request in the batch
+            # gen_kwargs = all_gen_kwargs[0]
+            breakpoint()
+            # Set default generation parameters if not provided
+            # if "max_new_tokens" not in gen_kwargs:
+            max_new_tokens = 8192
+            # if "temperature" not in gen_kwargs:
+            temperature = 0
+
+            self._sampling_params = SamplingParams(max_tokens=max_new_tokens, temperature=temperature, logprobs=1)
+            # TODO: for now, images and text are passed seperatly to the processor
+            assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
+            context = contexts[0]
+            visual = visuals[0]
+            
+            # TODO: handle multiple images / understand the `visuals` object
+            if not isinstance(visual, list):
+                visual = [visual]
+
+            # vLLM does not work with bytes, so we need to convert it to a data url
+            image_urls = [self.get_image_url(v) for v in visual]
+            
+            # image_url = self.get_image_url(visual)
+
+            # Pixtral expects inputs in a different format, and doesn't work with <image> tokens added in the middle of the prompt.
+            context = context.replace(DEFAULT_IMAGE_TOKEN, "")
+
+            if self.tag:
+                context = f"{self.tag} " + context
+
+            # create chat object
+            message = [
+                {"role": "user",
+                 "content": [{"type": "text", "text": context}] + [{"type": "image_url", "image_url": {"url": image_url}} for image_url in image_urls]
+                     }]
+            if self.add_system_prompt is not None:
+                message.insert(0, {"role": "system", "content": self.add_system_prompt})
+            
+            
+            
+            context_id = self._processor.apply_chat_template(message, padding=True, add_generation_prompt=False, tokenize=True, return_dict=True, return_tensors="pt").to(self._model.device)
+            
+            message.append({"role": "assistant", 
+                            "content": [{"type": "text", "text": continuation.strip()}]})
+            input = self._processor.apply_chat_template(message, padding=True, add_generation_prompt=False, tokenize=True, return_dict=True, return_tensors="pt").to(self._model.device)            
+            # labels = input["input_ids"].clone()
+            # labels[0, : context_id["input_ids"].shape[1]+2] = -100
+            # labels[0, -1:] = -100  # last token is also ignored
+            
+            
+            
+            
+            
+            try:
+                output = self._model.chat(message, sampling_params=self._sampling_params)
+                len_output_tokens = len(output[0].outputs[0].token_ids)
+                ce_loss = output[0].outputs[0].cumulative_logprob / len_output_tokens
+                result = (ce_loss, False)
+                # result = output[0].outputs[0].text
+            except Exception as e:
+                eval_logger.error(f"Error {e} in generating")
+                exit(1)
+            res.append(result)
+            pbar.update(1)
+        pbar.close()
+        return res
+
+
+
     def generate_until(self, requests: List[Instance]) -> List[str]:
         """Generate text based on the given requests."""
         res = []
diff --git a/lmms_eval/tasks/alm_bench/alm-bench-en.yaml b/lmms_eval/tasks/alm_bench/alm-bench-en.yaml
index 18a219a3c..dc118fc16 100644
--- a/lmms_eval/tasks/alm_bench/alm-bench-en.yaml
+++ b/lmms_eval/tasks/alm_bench/alm-bench-en.yaml
@@ -6,7 +6,7 @@ doc_to_visual: !function utils.alm_bench_doc_to_visual
 doc_to_text: !function utils.alm_bench_doc_to_text
 doc_to_target: !function utils.alm_bench_doc_to_target
 generation_kwargs:
-  max_new_tokens: 1024
+  max_new_tokens: 512
   temperature: 0
   do_sample: False
 process_results: !function utils.alm_bench_process_results
@@ -42,3 +42,7 @@ lmms_eval_specific_kwargs:
     prompt_format: "mcq"
     pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
     post_prompt: ""
+  llava_vllm:
+    prompt_format: "mcq"
+    pre_prompt: "For the given Multiple Choice Question Answer below, analyze the question and answer strictly from one of the options below. End your answer with 'Final Answer: <your answer>' where <your answer> is strictly the choice only."
+    post_prompt: ""
diff --git a/lmms_eval/tasks/alm_bench/utils.py b/lmms_eval/tasks/alm_bench/utils.py
index 008a1724c..d25bc087f 100644
--- a/lmms_eval/tasks/alm_bench/utils.py
+++ b/lmms_eval/tasks/alm_bench/utils.py
@@ -47,6 +47,7 @@ def alm_bench_doc_to_text(doc, lmms_eval_specific_kwargs):
 
 def alm_bench_process_results(doc, results):
     pred = extract_final_answer(results[0])
+    pred = re.sub(r" \n [\s\S]*$", "", pred)
     target, _ = split_answer_options(doc["Translated_Answer"])
     if target == None:
         print(doc["Translated_Answer"])
diff --git a/lmms_eval/tasks/commute/commute-ar.yaml b/lmms_eval/tasks/commute/commute-ar.yaml
index 2185ead36..ca8919345 100644
--- a/lmms_eval/tasks/commute/commute-ar.yaml
+++ b/lmms_eval/tasks/commute/commute-ar.yaml
@@ -13,7 +13,7 @@ doc_to_visual: !function utils.commute_doc_to_visual
 doc_to_text: !function utils.commute_doc_to_text
 process_results: !function utils.commute_process_results
 
-doc_to_target: "answer"
+doc_to_target: !function utils.commute_doc_to_target
 
 output_type: loglikelihood
 # generation_kwargs:
@@ -24,7 +24,7 @@ output_type: loglikelihood
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following source text from English to Arabic:\n{source}"
+    pre_prompt: "Translate the following source text from English to Arabic. Only generate the translation without any additional explanation or extra word.\nEnglish: {source}\n\nArabic: "
 
 metric_list:
   - metric: results
diff --git a/lmms_eval/tasks/commute/commute-cs.yaml b/lmms_eval/tasks/commute/commute-cs.yaml
index d659cb725..2f565b553 100644
--- a/lmms_eval/tasks/commute/commute-cs.yaml
+++ b/lmms_eval/tasks/commute/commute-cs.yaml
@@ -13,7 +13,7 @@ doc_to_visual: !function utils.commute_doc_to_visual
 doc_to_text: !function utils.commute_doc_to_text
 process_results: !function utils.commute_process_results
 
-doc_to_target: "answer"
+doc_to_target: !function utils.commute_doc_to_target
 
 output_type: loglikelihood
 # generation_kwargs:
@@ -24,7 +24,7 @@ output_type: loglikelihood
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following source text from English to Czech:\n{source}"
+    pre_prompt: "Translate the following source text from English to Czech. Only generate the translation without any additional explanation or extra word.\nEnglish: {source}\n\nCzech: "
 
 metric_list:
   - metric: results
diff --git a/lmms_eval/tasks/commute/commute-de.yaml b/lmms_eval/tasks/commute/commute-de.yaml
index f6f180dec..6014d5932 100644
--- a/lmms_eval/tasks/commute/commute-de.yaml
+++ b/lmms_eval/tasks/commute/commute-de.yaml
@@ -13,7 +13,7 @@ doc_to_visual: !function utils.commute_doc_to_visual
 doc_to_text: !function utils.commute_doc_to_text
 process_results: !function utils.commute_process_results
 
-doc_to_target: "answer"
+doc_to_target: !function utils.commute_doc_to_target
 
 output_type: loglikelihood
 # generation_kwargs:
@@ -24,7 +24,7 @@ output_type: loglikelihood
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following source text from English to German:\n{source}"
+    pre_prompt: "Translate the following source text from English to German. Only generate the translation without any additional explanation or extra word.\nEnglish: {source}\n\nGerman: "
 
 metric_list:
   - metric: results
diff --git a/lmms_eval/tasks/commute/commute-es.yaml b/lmms_eval/tasks/commute/commute-es.yaml
deleted file mode 100644
index ae6efeae5..000000000
--- a/lmms_eval/tasks/commute/commute-es.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
-dataset_path:  Unbabel/commute_multimodal_mt
-task: commute-es
-
-test_split: es
-
-
-dataset_kwargs:
-  token: True
-
-
-process_docs:  !function utils.commute_process_docs
-doc_to_visual: !function utils.commute_doc_to_visual
-doc_to_text: !function utils.commute_doc_to_text
-process_results: !function utils.commute_process_results
-
-doc_to_target: "answer"
-
-output_type: loglikelihood
-# generation_kwargs:
-#   max_new_tokens: 128
-#   temperature: 0
-#   do_sample: False
-
-
-lmms_eval_specific_kwargs:
-  default:
-    pre_prompt: "Translate the following source text from English to Spanish:\n{source}"
-
-metric_list:
-  - metric: results
-    aggregation: !function utils.commute_aggregate_results
-    higher_is_better: true
-
-metadata:
-  - version: 0.0
\ No newline at end of file
diff --git a/lmms_eval/tasks/commute/commute-fr.yaml b/lmms_eval/tasks/commute/commute-fr.yaml
index e302c9944..49028b961 100644
--- a/lmms_eval/tasks/commute/commute-fr.yaml
+++ b/lmms_eval/tasks/commute/commute-fr.yaml
@@ -13,7 +13,7 @@ doc_to_visual: !function utils.commute_doc_to_visual
 doc_to_text: !function utils.commute_doc_to_text
 process_results: !function utils.commute_process_results
 
-doc_to_target: "answer"
+doc_to_target: !function utils.commute_doc_to_target
 
 output_type: loglikelihood
 # generation_kwargs:
@@ -24,7 +24,7 @@ output_type: loglikelihood
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following source text from English to French:\n{source}"
+    pre_prompt: "Translate the following source text from English to French. Only generate the translation without any additional explanation or extra word.\nEnglish: {source}\n\nFrench: "
 
 metric_list:
   - metric: results
diff --git a/lmms_eval/tasks/commute/commute-ru.yaml b/lmms_eval/tasks/commute/commute-ru.yaml
index db9ad045f..15d73fe8f 100644
--- a/lmms_eval/tasks/commute/commute-ru.yaml
+++ b/lmms_eval/tasks/commute/commute-ru.yaml
@@ -13,7 +13,7 @@ doc_to_visual: !function utils.commute_doc_to_visual
 doc_to_text: !function utils.commute_doc_to_text
 process_results: !function utils.commute_process_results
 
-doc_to_target: "answer"
+doc_to_target: !function utils.commute_doc_to_target
 
 output_type: loglikelihood
 # generation_kwargs:
@@ -24,7 +24,7 @@ output_type: loglikelihood
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following source text from English to Russian:\n{source}"
+    pre_prompt: "Translate the following source text from English to Russian. Only generate the translation without any additional explanation or extra word.\nEnglish: {source}\n\nRussian: "
 
 metric_list:
   - metric: results
diff --git a/lmms_eval/tasks/commute/commute-zh.yaml b/lmms_eval/tasks/commute/commute-zh.yaml
index d8141d79b..4ef21e501 100644
--- a/lmms_eval/tasks/commute/commute-zh.yaml
+++ b/lmms_eval/tasks/commute/commute-zh.yaml
@@ -13,7 +13,7 @@ doc_to_visual: !function utils.commute_doc_to_visual
 doc_to_text: !function utils.commute_doc_to_text
 process_results: !function utils.commute_process_results
 
-doc_to_target: "answer"
+doc_to_target: !function utils.commute_doc_to_target
 
 output_type: loglikelihood
 # generation_kwargs:
@@ -24,7 +24,7 @@ output_type: loglikelihood
 
 lmms_eval_specific_kwargs:
   default:
-    pre_prompt: "Translate the following source text from English to Chinese:\n{source}"
+    pre_prompt: "Translate the following source text from English to Chinese (Simlified). Only generate the translation without any additional explanation or extra word.\nEnglish: {source}\n\nChinese (Simplified): "
 
 metric_list:
   - metric: results
diff --git a/lmms_eval/tasks/commute/utils.py b/lmms_eval/tasks/commute/utils.py
index 938a0aec1..90c02e1fd 100644
--- a/lmms_eval/tasks/commute/utils.py
+++ b/lmms_eval/tasks/commute/utils.py
@@ -50,9 +50,11 @@ def commute_doc_to_text(doc,lmms_eval_specific_kwargs=None ):
     source_txt = doc["source"]
     pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
     pre_prompt = pre_prompt.format(source=source_txt)
-    input = pre_prompt+doc["output"]
-    return f"{input}"
+    return f"{pre_prompt}"    
 
+def commute_doc_to_target(doc, model_specific_target_kwargs=None):
+    # The target is the correct translation
+    return doc["output"]
 
 def commute_process_results(doc, results):
     loglikelihood = results[0][0]

From d04bc31e21ad9c969dedafadf9ee6d6063e11baa Mon Sep 17 00:00:00 2001
From: farajian <amin@unbabel.com>
Date: Mon, 9 Jun 2025 16:56:08 +0000
Subject: [PATCH 109/112] update for pixtral

---
 lmms_eval/models/pixtral.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lmms_eval/models/pixtral.py b/lmms_eval/models/pixtral.py
index c6e60ef72..ecccc3219 100644
--- a/lmms_eval/models/pixtral.py
+++ b/lmms_eval/models/pixtral.py
@@ -166,7 +166,7 @@ def config(self):
 
     @property
     def tokenizer(self):
-        return self._tokenizer
+        return self._model.get_tokenizer()
 
     @property
     def model(self):
@@ -264,7 +264,7 @@ def batch_requests(requests, batch_size):
             # if "temperature" not in gen_kwargs:
             temperature = 0
 
-            self._sampling_params = SamplingParams(max_tokens=max_new_tokens, temperature=temperature, logprobs=1)
+            self._sampling_params = SamplingParams(max_tokens=max_new_tokens, temperature=temperature, logprobs=1, prompt_logprobs=1)
             # TODO: for now, images and text are passed seperatly to the processor
             assert self.batch_size_per_gpu == 1, "Do not support batch_size_per_gpu > 1 for now"
             context = contexts[0]
@@ -288,7 +288,7 @@ def batch_requests(requests, batch_size):
             # create chat object
             message = [
                 {"role": "user",
-                 "content": [{"type": "text", "text": context}] + [{"type": "image_url", "image_url": {"url": image_url}} for image_url in image_urls]
+                 "content": [{"type": "text", "text": context}]
                      }]
             if self.add_system_prompt is not None:
                 message.insert(0, {"role": "system", "content": self.add_system_prompt})
@@ -297,6 +297,7 @@ def batch_requests(requests, batch_size):
             
             context_id = self._processor.apply_chat_template(message, padding=True, add_generation_prompt=False, tokenize=True, return_dict=True, return_tensors="pt").to(self._model.device)
             
+            message.insert(1, [{"type": "image_url", "image_url": {"url": image_url}} for image_url in image_urls])
             message.append({"role": "assistant", 
                             "content": [{"type": "text", "text": continuation.strip()}]})
             input = self._processor.apply_chat_template(message, padding=True, add_generation_prompt=False, tokenize=True, return_dict=True, return_tensors="pt").to(self._model.device)            

From 3c287bb9726a83507fbc5384217bc157c7bcc3e7 Mon Sep 17 00:00:00 2001
From: manzar96 <manoszar96@gmail.com>
Date: Mon, 9 Jun 2025 19:35:28 +0000
Subject: [PATCH 110/112] llavq_hd loglikelihood implementations + changes on
 commute

---
 lmms_eval/models/llava_hf.py                  |  13 +-
 lmms_eval/tasks/commute/comet_utils/comet.py  | 114 ----
 .../commute/comet_utils/metrics_handler.py    |  75 ---
 .../tasks/commute/comet_utils/myutils.py      | 611 ------------------
 .../commute/comet_utils/result_handler.py     |  39 --
 lmms_eval/tasks/commute/utils.py              |  10 -
 6 files changed, 6 insertions(+), 856 deletions(-)
 delete mode 100644 lmms_eval/tasks/commute/comet_utils/comet.py
 delete mode 100644 lmms_eval/tasks/commute/comet_utils/metrics_handler.py
 delete mode 100644 lmms_eval/tasks/commute/comet_utils/myutils.py
 delete mode 100644 lmms_eval/tasks/commute/comet_utils/result_handler.py

diff --git a/lmms_eval/models/llava_hf.py b/lmms_eval/models/llava_hf.py
index 0dcc4698f..30a1e1a36 100644
--- a/lmms_eval/models/llava_hf.py
+++ b/lmms_eval/models/llava_hf.py
@@ -226,13 +226,12 @@ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
                 prompt_and_continuation = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
             formatted_contexts = [prompt]
             formatted_continuation = [prompt_and_continuation]
+            model_inputs_wo_continuation = self._image_processor(text=formatted_contexts, images=visuals, return_tensors="pt").to(self._device, self.model.dtype)
             model_inputs = self._image_processor(text=formatted_continuation, images=visuals, return_tensors="pt").to(self._device, self.model.dtype)
             labels = model_inputs["input_ids"].clone()
-            contxt_id = self._image_processor(text=formatted_contexts, return_tensors="pt")["input_ids"]
-            breakpoint()
-            labels[:, : contxt_id.shape[1]] = -100
-            labels[0, -1:] = -100  # last token is also ignored
-
+            image_context_shape = model_inputs_wo_continuation["input_ids"].shape
+            labels[:, : image_context_shape[1]] = -100
+            # labels[0, -1:] = -100  # last token is also ignored
             if self.accelerator.is_main_process and doc_id % 100 == 0:
                 eval_logger.debug(f"Prompt for doc ID {doc_id}:\n\n{formatted_contexts[0]}\n")
                 eval_logger.debug(f"Prompt and continuation for doc ID {doc_id}:\n\n{formatted_continuation[0]}\n")
@@ -242,8 +241,8 @@ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
             loss = outputs["loss"]
             logits = outputs["logits"]
             greedy_tokens = logits.argmax(dim=-1)
-            cont_toks = model_inputs["input_ids"][:, contxt_id.shape[1] :]  # [1, seq]
-            greedy_tokens = greedy_tokens[:, contxt_id.shape[1] : model_inputs["input_ids"].shape[1]]  # [1, seq]
+            cont_toks = model_inputs["input_ids"][:, image_context_shape[1] :]  # [1, seq]
+            greedy_tokens = greedy_tokens[:, image_context_shape[1] : model_inputs["input_ids"].shape[1]]  # [1, seq]
             max_equal = (greedy_tokens == cont_toks).all()
             res.append((float(loss.item()), bool(max_equal)))
             pbar.update(1)
diff --git a/lmms_eval/tasks/commute/comet_utils/comet.py b/lmms_eval/tasks/commute/comet_utils/comet.py
deleted file mode 100644
index e5cfe1faa..000000000
--- a/lmms_eval/tasks/commute/comet_utils/comet.py
+++ /dev/null
@@ -1,114 +0,0 @@
-# -*- coding: utf-8 -*-
-from comet import download_model, load_from_checkpoint
-
-from lmms_eval.tasks.commute.comet_utils.metrics_handler import Metric
-from lmms_eval.tasks.commute.comet_utils.result_handler import MetricResult
-
-
-class BaseCOMETResult(MetricResult):
-    """
-    COMET Result Handler.
-    """
-
-    def __init__(
-        self,
-        result: float,
-    ) -> None:
-        super().__init__(result)
-
-
-class BaseCOMET(Metric):
-    def __init__(self, model: str, **kwargs) -> None:
-        super().__init__(**kwargs)
-        model_path = download_model(model)
-        self.model = load_from_checkpoint(model_path)
-        self.model.eval()
-
-    def load_gold_data(self, gold_data):
-        pass
-
-    def make_samples(
-        self, sources: list[str], hypotheses: list[str], references: list[str] = None
-    ):
-        pass
-
-    def run(
-        self,
-        hypothesis_path,
-        gold_data_path,
-        gpus: int = 1,
-        batch_size: int = 16,
-        **kwargs
-    ) -> dict:
-        hypotheses, gold_data = self._handle_inputs(hypothesis_path, gold_data_path)
-        references, sources = self.load_gold_data(gold_data)
-        result = self.evaluate(hypotheses, references, sources, gpus, batch_size)
-        result.print_result(self.metric_name())
-        return result.format_result(self.metric_name())
-
-    def evaluate(
-        self, hypotheses: list, references: list, sources: list, gpus, batch_size
-    ) -> BaseCOMETResult:
-        """
-        Evaluate function receives the hypotheses and the references and returns a COMETResult object.
-
-        :param hypotheses: List of the MT outputs (sentences).
-        :param references: List of the reference sentences.
-        :param sources: List of source sentences
-        """
-        samples = self.make_samples(sources, hypotheses, references)
-
-        outputs = self.model.predict(
-            samples=samples,
-            batch_size=batch_size,
-            gpus=gpus,
-            accelerator="auto",
-        )
-        system_score, segments_scores = outputs.system_score, outputs.scores
-
-        comet_result = BaseCOMETResult(
-            {
-                "system_score": system_score,
-                "segments_scores": segments_scores,
-            }
-        )
-        return comet_result
-
-    def process_result(self, result) -> MetricResult:
-        pass
-
-    @staticmethod
-    def metric_name():
-        pass
-
-
-class RefCOMET(BaseCOMET):
-    def __init__(self, model: str, **kwargs) -> None:
-        super().__init__(model, **kwargs)
-
-    def load_gold_data(self, gold_data):
-        references, sources = gold_data["ref"], gold_data["src"]
-        return references, sources
-
-    def make_samples(
-        self, sources: list[str], hypotheses: list[str], references: list[str]
-    ):
-        samples = {"src": sources, "mt": hypotheses, "ref": references}
-        samples = [dict(zip(samples, t)) for t in zip(*samples.values())]
-        return samples
-
-
-class QECOMET(BaseCOMET):
-    def __init__(self, model: str, **kwargs) -> None:
-        super().__init__(model, **kwargs)
-
-    def load_gold_data(self, gold_data):
-        references, sources = None, gold_data["src"]
-        return references, sources
-
-    def make_samples(
-        self, sources: list[str], hypotheses: list[str], references: list[str] = None
-    ):
-        samples = {"src": sources, "mt": hypotheses}
-        samples = [dict(zip(samples, t)) for t in zip(*samples.values())]
-        return samples
\ No newline at end of file
diff --git a/lmms_eval/tasks/commute/comet_utils/metrics_handler.py b/lmms_eval/tasks/commute/comet_utils/metrics_handler.py
deleted file mode 100644
index 29333f9d4..000000000
--- a/lmms_eval/tasks/commute/comet_utils/metrics_handler.py
+++ /dev/null
@@ -1,75 +0,0 @@
-# -*- coding: utf-8 -*-
-from abc import ABC, abstractmethod
-from pathlib import Path
-
-from lmms_eval.tasks.commute.comet_utils.result_handler import MetricResult
-from lmms_eval.tasks.commute.comet_utils.myutils import list_to_dict, load_jsonl_file, read_lines
-
-
-class Metric(ABC):
-    """Abstract class defining a shared interface for all the Metrics"""
-
-    def __init__(self, **kwargs) -> None:
-        self.kwargs = kwargs
-
-    def run(self, hypothesis_path: Path = None, references_path: Path = None, **kwargs):
-        """
-        The runner function that performs all the necessary checks on the files,
-        applies the needed preprocessing (like lowercasing, tokenization, etc),
-        and calls the evaluate method of the class to get the scores.
-        """
-        pass
-
-    @abstractmethod
-    def evaluate(self, hypotheses: list, references: list, **kwargs) -> MetricResult:
-        """
-        Evaluate function receives the hypotheses and the reference files and returns a MetricResult object.
-
-        :param hypotheses: the path to the hypothese file.
-        :param references: the path to the reference file.
-        :return: MetricResult object.
-        """
-        pass
-
-    @abstractmethod
-    def process_result(self, result) -> MetricResult:
-        """
-        Process the result to be ready and complient with the MetricResult format.
-
-        :param result: the raw result produced by the metric
-        :return: MetricResult object.
-        """
-        pass
-
-    @staticmethod
-    @abstractmethod
-    def metric_name() -> None:
-        """Metric name used to address the metric via cli."""
-        pass
-
-    @staticmethod
-    def _handle_inputs(
-        hypotheses: Path,
-        data_path: Path,
-    ) -> tuple:
-        """
-        Function to handle input files.
-        All inputs will be returned as list of strings.
-
-        :param hypotheses: either the handler to the file storing the hypotheses.
-        :param references: either the handler to the file storing the refereneces.
-        :param sources: either the handler to the file storing the source sentences.
-
-        :return:
-            -  Tuple with hypotheses, references and kwargs
-            -  If sources not None, Tuple with hypotheses, references, sources and kwargs
-        """
-        hypotheses = read_lines(hypotheses, unescape_newline=True)
-        # gold data keys depend on the task; e.g., for MT, it will include "ref", for APE "pe"
-        gold_data = load_jsonl_file(data_path)
-        assert len(hypotheses) == len(
-            gold_data
-        ), f"The number of hypotheses ({len(hypotheses)}) and rows in the gold data ({len(gold_data)}) should be the same."
-        gold_data = list_to_dict(gold_data)
-
-        return hypotheses, gold_data
\ No newline at end of file
diff --git a/lmms_eval/tasks/commute/comet_utils/myutils.py b/lmms_eval/tasks/commute/comet_utils/myutils.py
deleted file mode 100644
index 4adf7eaef..000000000
--- a/lmms_eval/tasks/commute/comet_utils/myutils.py
+++ /dev/null
@@ -1,611 +0,0 @@
-import argparse
-import ast
-import json
-import os
-import re
-import subprocess
-import time
-from pathlib import Path
-from typing import Dict, Iterable, List, Union
-
-import lxml
-import numpy as np
-import spacy
-import tqdm
-import yaml
-from loguru import logger
-from lxml import etree
-# from mosestokenizer import MosesDetokenizer
-from nltk.tokenize.treebank import TreebankWordDetokenizer
-from tenacity import Retrying, stop_after_attempt, wait_random_exponential
-
-PathInput = Union[str, Path]
-PATTERN_OPEN_TAG = r"<\s*([\w]+)\s*>\s*"
-PATTERN_CLOSE_TAG = r"\s*<\s*\/\s*([\w]+)\s*>"
-PATTERN_SHOT_NAME = r"^([0-9]+)[-_]shot.*$"
-
-SPACY_LANGUAGE_TO_MODEL = {
-    "en": "en_core_web_sm",
-    "de": "de_core_news_sm",
-    "fr": "fr_core_news_sm",
-    "nl": "nl_core_news_sm",
-    "pt": "pt_core_news_sm",
-    "ru": "ru_core_news_sm",
-    "it": "it_core_news_sm",
-    "pl": "pl_core_news_sm",
-    "es": "es_core_news_sm",
-    "sv": "sv_core_news_sm",
-    "zh": "zh_core_web_sm",
-    "ko": "ko_core_news_sm",
-}
-
-
-def args_to_dict(args, prefix: str, strip_prefix: bool = False):
-    """Filters argparse's `Namespace` into dictionary with arguments
-    beginning with the given prefix."""
-    prefix += "_"
-    d = {}
-    for k, v in args.__dict__.items():
-        if k.startswith(prefix) or k in ["language", "tokenize", "lowercase"]:
-            k = k.replace(prefix, "") if strip_prefix else k
-            d[k] = v
-    return d
-
-
-def detokenize(lines: list[str]) -> list[str]:
-    detokenizer = TreebankWordDetokenizer()
-    detokenized_lines = [detokenizer.detokenize(line.split()) for line in lines]
-    return detokenized_lines
-
-
-def detokenize_moses(lines: list[str], lang: str) -> list[str]:
-    detokenizer = MosesDetokenizer(lang)
-    detokenized_lines = [detokenizer(line.split()) for line in lines]
-    return detokenized_lines
-
-
-def tokenize_spacy(lines: list[str], language, keep_xml=False) -> list[str]:
-    spacy_model = SPACY_LANGUAGE_TO_MODEL[language]
-    try:
-        tokenizer = spacy.load(spacy_model, disable=["tagger", "parser", "ner"])
-    except OSError:
-        logger.warning(f"Spacy model {spacy_model} not found. Trying to download...")
-        download_proc = subprocess.run(
-            ["python", "-m", "spacy", "download", spacy_model]
-        )
-        tokenizer = spacy.load(spacy_model, disable=["tagger", "parser", "ner"])
-
-    tokenized = list(tokenizer.pipe(lines))
-    tokenized = [[token.text for token in sentence] for sentence in tokenized]
-    output_lines = [" ".join(sentence) for sentence in tokenized]
-    if keep_xml:
-        output_lines = [
-            re.sub(PATTERN_OPEN_TAG, r"<\1>", output_line)
-            for output_line in output_lines
-        ]
-        output_lines = [
-            re.sub(PATTERN_CLOSE_TAG, r"</\1>", output_line)
-            for output_line in output_lines
-        ]
-    return output_lines
-
-
-def parse_yaml_config(config_path):
-    with open(config_path, "r") as config_file:
-        config_data = yaml.safe_load(config_file)
-    return config_data
-
-
-def get_task_args(args: dict) -> dict:
-    source = args.get("source")
-    hypothesis = args.get("hypothesis")
-    references = args.get("references")
-
-    task_args = {
-        "hypothesis": hypothesis,
-        "references": references,
-    }
-    if source:
-        task_args.update({"source": source})
-    return task_args
-
-
-def load_json_file(
-    file_path: PathInput,
-) -> list:
-    """
-    Function to to load json files.
-
-    :param file_path: either the handler to the file storing the hypotheses.
-    """
-    with open(file_path, "r", encoding="utf-8") as f:
-        return json.load(f)
-
-
-def load_jsonl_file(
-    file_path: PathInput,
-) -> list:
-    """
-    Function to parse error span jsonl files.
-    All inputs will be returned as list of dictionaries.
-
-    :param file_path: either the handler to the file storing the hypotheses.
-        Must be a jsonl file where each line is a dict which has at least the following keys:
-        {"mt": <mt>, "annotations": [{"start": <start>, "end": <end>, "severity": <severity>}, ...]}
-    :return:
-        -  List of dictionaries
-    """
-    with open(file_path, "r", encoding="utf-8") as f:
-        json_list = list(f)
-
-    return [json.loads(l) for l in json_list]
-
-
-def list_to_dict(l: list[dict[str, str]]) -> dict[str, list[str]]:
-    """
-    Assumes lists in dict have the same length.
-    """
-    d = {}
-    for k in l[0].keys():
-        d[k] = [d[k] for d in l]
-    return d
-
-
-def dict_to_records(d: dict[str, list[str]]) -> list[dict[str, str]]:
-    """
-    Assumes lists in dict have the same length.
-    """
-    records = []
-    for i in range(len(list(d.values())[0])):
-        records.append({k: v[i] for k, v in d.items()})
-
-    return records
-
-
-def write_lines(
-    path: PathInput,
-    lines: Iterable[str],
-    escape_newline: bool = False,
-    escape_return_char: bool = True,
-    verbose: bool = True,
-) -> None:
-    """Writes lines to a file.
-
-    Lines can be escaped, meaning \n is transformed to \\n.
-
-    Args:
-        path: The path to the file.
-        lines: The lines to write.
-        escape_newline: Whether to escape newlines.
-    """
-    # make dir, if not exists
-    if isinstance(path, str):
-        path = Path(path)
-    path.parent.mkdir(parents=True, exist_ok=True)
-    out_lines = []
-    for i, line in enumerate(lines):
-        if escape_return_char:
-            if "\r" in line and verbose:
-                logger.opt(ansi=True).warning(
-                    f"Detected carriage return in line {i + 1} (\\r). This may cause errors downstream. Escaping. This behaviour is the default; you can turn it off with escape_return_char."
-                )
-            line = line.replace("\r", "\\r")
-        if escape_newline:
-            if "\n" in line and verbose:
-                logger.opt(ansi=True).warning(
-                    f"Found new line in line {i + 1} (\\n). This may cause errors downstream. Escaping."
-                )
-            line = line.replace("\n", "\\n")
-        out_lines.append(line)
-    with open(path, "w", encoding="utf-8") as f:
-        f.writelines((f"{l}\n" for l in out_lines))
-
-
-def combine_metrics_args(task_metric: dict, subtask_metric: dict) -> dict:
-    eval_metric = task_metric.copy()
-    for metric_arg, metric_arg_value in subtask_metric.items():
-        eval_metric[metric_arg] = metric_arg_value
-    return eval_metric
-
-
-def generate_with_retries(
-    retry_function,
-    model_args,
-    retry_max_attempts=1,
-    retry_multiplier=1,
-    retry_max_interval=10,
-    retry_min_interval=4,
-):
-    retryer = Retrying(
-        stop=stop_after_attempt(retry_max_attempts),
-        wait=wait_random_exponential(
-            multiplier=retry_multiplier, max=retry_max_interval, min=retry_min_interval
-        ),
-        reraise=True,
-    )
-    return retryer(retry_function, **model_args)
-
-
-def sample_strings_from_list(strings: list[str], num_samples: int) -> list[str]:
-    """ """
-    return np.random.choice(strings, num_samples).tolist()
-
-
-def read_lines(path: PathInput, unescape_newline: bool = False) -> List[str]:
-    """Reads lines from a file.
-    Lines can be unescapped, meaning \\n is transformed to \n.
-    Args:
-        path: The path to the file.
-        unescape_newline: Whether to unescape newlines.
-    Returns:
-        The lines in the file."""
-    with open(path, encoding="utf-8") as f:
-        lines = [l[:-1] for l in f.readlines()]
-    if unescape_newline:
-        lines = [l.replace("\\n", "\n") for l in lines]
-    return lines
-
-
-def make_dir_if_not_exists(output_file):
-    output_dir = os.path.dirname(output_file)
-    if not os.path.exists(output_dir):
-        os.makedirs(output_dir)
-
-
-def log_response(response: str, step: int, lim: int = 10) -> None:
-    """
-    Logs the response from the model.
-    """
-    if step < lim:
-        if "<" in response:
-            response = response.replace("<", "\\<")
-            logger.opt(colors=True).info(
-                f"<red> Escaped tags in below response to not throw colorizing error. </red>"
-            )
-        logger.opt(colors=True).info(f"Response {step}: {response.strip()}")
-
-
-def get_num_processed_lines(output_file: str, **kwargs):
-    """
-    This function is used to write the output from the last line of the input file.
-    """
-    if os.path.exists(output_file):
-        processed_lines = read_lines(output_file, unescape_newline=False)
-        num_processed_lines = len(processed_lines)
-        if num_processed_lines > 0:
-            logger.opt(ansi=True).warning(
-                f"<red>Resuming generation from line {num_processed_lines}.</red>"
-            )
-    else:
-        num_processed_lines = 0
-
-    return num_processed_lines
-
-
-def load_data_to_records(
-    data_path: str,
-) -> list[dict[str, str]]:
-    """ """
-    # prompt data file can either be a jsonl or a csv
-    data = load_jsonl_file(data_path)
-
-    return data
-
-
-def save_to_json(
-    save_location: PathInput,
-    data: dict,
-) -> None:
-    """ """
-    if isinstance(save_location, str):
-        save_location = Path(save_location)
-    save_location.parent.mkdir(parents=True, exist_ok=True)
-    with open(save_location, "w") as output_path:
-        json.dump(data, output_path, indent=4)
-
-
-def get_sacrebleu_segment_scores(
-    hypotheses: list[str],
-    references,
-    method,
-) -> list[float]:
-    """ """
-    segment_scores = []
-    for h, r in zip(hypotheses, references):
-        segment_score = method.sentence_score(h, r)
-        segment_scores.append(segment_score.score)
-    return segment_scores
-
-
-def get_eval_args_given_task(
-    eval_args: Dict[str, str],
-    task_name: str,
-    subtask: str,
-    gen_output_dir: Path,
-    eval_data_dir: Path,
-    eval_output_dir: Path,
-    model_type: str,
-    model_name: str,
-) -> Dict[str, str]:
-    """ """
-    if task_name == "gec":
-        eval_args["references_m2"] = eval_data_dir / task_name / subtask / "test_corpus.m2"
-    gen_output_path = (
-        gen_output_dir
-        / task_name
-        / subtask
-        / model_type
-        / model_name
-        / "generation.txt"
-    )
-
-    eval_output_path = (
-        eval_output_dir
-        / task_name
-        / subtask
-        / model_type
-        / model_name
-        / "evaluation.json"
-    )
-
-    eval_data_path = eval_data_dir / task_name / subtask / "test.jsonl"
-    # add language argument to eval_args as it is needed in some metrics
-    lp = subtask.split(".")[1]
-    src_lang, trg_lang = get_langs(lp)
-    eval_args["lp"] = {"src_lang": src_lang, "trg_lang": trg_lang}
-
-    return gen_output_path, eval_data_path, eval_output_path, eval_args
-
-
-def text_to_label(
-    text: str,
-    text_type: str,
-    label_space: List[str] = None,
-    return_is_random: bool = False,
-) -> int:
-    is_random = False
-    if text_type == "categorical":
-        label = int(text.strip())
-        is_random = False
-    # if hypothesis is text
-    elif text_type == "text":
-        label = np.random.randint(len(label_space))
-        is_random = True
-        for l in label_space:
-            if re.search(l, text, re.IGNORECASE) is not None:
-                label = label_space.index(l)
-                is_random = False
-                break
-    if return_is_random:
-        return label, is_random
-    else:
-        return label
-
-
-def handle_subprocess(subprocess_args: List[str], check_output: bool = False):
-    output = 1
-    if check_output:
-        process = subprocess.Popen(
-            subprocess_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
-        )
-        output = ""
-
-        # Read and print the output in real-time
-        for line in iter(process.stdout.readline, ""):
-            print(line, end="")  # Print each line as it's received
-            output += line  # Append each line to the output variable
-    else:
-        try:
-            process = subprocess.Popen(subprocess_args)
-            while process.poll() is None:
-                time.sleep(1)
-            logger.info("Generation process has finished.")
-            output = 0
-        except KeyboardInterrupt:
-            process.terminate()
-    return output
-
-
-def tokenize_text(references, tokenizer):
-    tokenized_prompts = []
-    logger.info(f"Tokenizing text...")
-    for reference in tqdm.tqdm(references):
-        tokenized_prompts.append(tokenizer.encode(reference))
-    return tokenized_prompts
-
-
-def get_langs(lp):
-    # TODO: This has to be updated to account for more languages
-    valid_langs = [
-        "de",
-        "en-us",
-        "en-gb",
-        "en-uk",
-        "en",
-        "es-latam",
-        "es",
-        "fr",
-        "fr-ca",
-        "fr-fr",
-        "it",
-        "it-it",
-        "ko",
-        "ko-ko",
-        "nl",
-        "nl-nl",
-        "pl",
-        "pl-pl",
-        "pt-br",
-        "pt",
-        "pt-pt",
-        "ru",
-        "zh-CN",
-        "zh-cn",
-        "zh-TW",
-        "zh-tw",
-        "zh",
-        "ja",
-        "hi",
-        "uk",
-        "cs",
-        "is",
-        "sv",
-        "sv-se",
-        "ar",
-        "tr",
-        "el",
-        "hu",
-        "hu-hu",
-        "ro",
-        "ro-ro",
-        "fi",
-        "fi-fi",
-        "sk",
-        "da",
-        "da-da",
-        "lt",
-        "lv",
-        "sl",
-        "et",
-        "bg",
-        "no",
-        "no-no",
-        "ca",
-        "hr",
-        "ga",
-        "mt",
-        "gl",
-    ]
-    lang_pattern = "|".join(valid_langs)
-    lp_pattern = rf"^({lang_pattern})-({lang_pattern})$"
-    match = re.match(lp_pattern, lp)
-    if match:
-        # We have a language pair, so both src_lang and trg_lang will be extracted from lp
-        src_lang = match.group(1)
-        trg_lang = match.group(2)
-        return src_lang, trg_lang
-    elif lp in valid_langs:
-        # the task is monolingual, hence we only have one language. So, we set the src_lang only. trg_lang will be set to None
-        src_lang = lp
-        trg_lang = None
-        return src_lang, trg_lang
-    return None, None
-
-
-def add_average_generation_time(
-    input_file: str, metadata_file: str, language: str, mode: str = "lps"
-):
-    """
-    Given the generation file, the metadata file and the mode it calculates the time to generate the outputs.
-    Depending on the mode, it can be either the lines-per-second (lps) or words-per-second (wps)
-
-    Args:
-        input_file (str): The generation file
-        metadata_file (str): The metadata file that contains the generation_time which is a list
-        mode (str, optional): The mode for calculating the time. It can be either the lines-per-second (lps) or words-per-second (wps). Defaults to "lps".
-    """
-    lines = read_lines(input_file)
-    metadata = load_json_file(metadata_file)
-    generation_time = metadata.get("generation_time")
-    if generation_time:
-        total_time = sum(generation_time)
-        if mode == "lps":
-            average_time = len(lines) / total_time
-        # TODO: Add the support of wps mode
-    else:
-        logger.error(
-            f"The generation_time information doesn't exist in the {metadata_file}. We are going to set the average generation time to -1."
-        )
-        average_time = -1
-    metadata[f"generation_time_average ({mode})"] = average_time
-    save_to_json(metadata_file, metadata)
-
-
-def parse_dict_arg(arg):
-    try:
-        # Safely evaluate the string as a Python expression
-        parsed = ast.literal_eval(arg)
-        if isinstance(parsed, dict):
-            return parsed
-        else:
-            raise argparse.ArgumentTypeError("Argument is not a valid dictionary")
-    except (ValueError, SyntaxError):
-        raise argparse.ArgumentTypeError("Argument is not a valid dictionary")
-
-
-def write_jsonl(path, data):
-    with open(path, "w") as f:
-        for d in data:
-            f.write(json.dumps(d, ensure_ascii=False) + "\n")
-
-
-def match_xml(string_1: lxml.etree._Element, string_2: lxml.etree._Element):
-    if string_1.tag != string_2.tag:
-        return False
-
-    string_1 = list(string_1.iterchildren())
-    string_2 = list(string_2.iterchildren())
-
-    if len(string_1) != len(string_2):
-        return False
-
-    for s1, s2 in zip(string_1, string_2):
-        if not match_xml(s1, s2):
-            return False
-    return True
-
-
-def split_by_xml_tags(string: str):
-    try:
-        root = etree.fromstring(f"<root>{string}</root>")
-        parts = []
-
-        for element in root.iter():
-            if element.tag == "root":
-                if element.text:
-                    parts.append(element.text)
-                continue
-
-            if element.text:
-                parts.append(element.text)
-            if element.tail:
-                parts.append(element.tail)
-
-        return [p.strip() for p in parts if p.strip()]
-    except etree.XMLSyntaxError:
-        return [string]
-
-
-def split_pairs_by_xml_tags(string1: str, string2: str):
-    try:
-        string1_xml = etree.fromstring(f"<root>{string1}</root>")
-        string2_xml = etree.fromstring(f"<root>{string2}</root>")
-        if not match_xml(string1_xml, string2_xml):
-            return (None, None)
-
-        string1_segmented = split_by_xml_tags(string1)
-        string2_segmented = split_by_xml_tags(string2)
-        string1_segmented = [seg for seg in string1_segmented if seg is not None]
-        string2_segmented = [seg for seg in string2_segmented if seg is not None]
-
-        if len(string1_segmented) == len(string2_segmented):
-            return (string1_segmented, string2_segmented)
-    except:
-        pass
-
-    return (None, None)
-
-def prepare_xml_markup_pairs(hypotheses, references):
-    hypothesis_segmented, references_segmented = [], []
-    non_matching_indices = []
-    current_index = 0
-    for hyp, ref in zip(hypotheses, references):
-        hs, rs = split_pairs_by_xml_tags(hyp, ref)
-        if hs and rs:
-            hypothesis_segmented.extend(hs)
-            references_segmented.extend(rs)
-            current_index += len(hs)
-        else:
-            non_matching_indices.append(current_index)
-            current_index += 1
-    return hypothesis_segmented, references_segmented, non_matching_indices
\ No newline at end of file
diff --git a/lmms_eval/tasks/commute/comet_utils/result_handler.py b/lmms_eval/tasks/commute/comet_utils/result_handler.py
deleted file mode 100644
index 9b571d909..000000000
--- a/lmms_eval/tasks/commute/comet_utils/result_handler.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# -*- coding: utf-8 -*-
-from abc import ABC
-
-
-class MetricResult(ABC):
-    """
-    Abstract class defining a shared interface for all Metric Result Handlers.
-
-    :param result: float value to be displayed.
-    """
-
-    def __init__(self, result: float) -> None:
-        self.result = result
-
-    def print_result(self, metric_name: str, round_to_decimals: int = 4) -> None:
-        """Function used to display a particular Metric result.
-        :param metric_name: Metric name.
-        :param round_to_decimals: decimals that we want to present.
-        """
-        if type(self.result) == dict:
-            print(
-                f'{metric_name}: {round(self.result["system_score"], round_to_decimals)}'
-            )
-        else:
-            print(f"{metric_name}: {round(self.result, round_to_decimals)}")
-
-    def format_result(self, metric_name: str, round_to_decimals: int = 4) -> dict:
-        """Function used to format a particular Metric result.
-        :param metric_name: Metric name.
-        :param round_to_decimals: decimals that we want to present.
-        """
-        if type(self.result) == dict:
-            out = {
-                f"{metric_name}": round(self.result["system_score"], round_to_decimals),
-                f"{metric_name}_segments": self.result["segments_scores"],
-            }
-        else:
-            out = {f"{metric_name}": round(self.result, round_to_decimals)}
-        return out
\ No newline at end of file
diff --git a/lmms_eval/tasks/commute/utils.py b/lmms_eval/tasks/commute/utils.py
index 90c02e1fd..e6937a9d0 100644
--- a/lmms_eval/tasks/commute/utils.py
+++ b/lmms_eval/tasks/commute/utils.py
@@ -7,16 +7,6 @@
 
 from loguru import logger
 
-from lmms_eval.tasks.commute.comet_utils.comet import RefCOMET
-
-
-# def base64_to_bytes(base64_string):
-#     # Remove the header if it exists (e.g., "data:image/jpeg;base64,")
-#     if "base64," in base64_string:
-#         base64_string = base64_string.split("base64,")[1]
-#     # Decode base64 string to bytes
-#     img_bytes = base64.b64decode(base64_string)
-#     return img_bytes
 
 def commute_process_docs(docs):
 

From f8e35c336ab36fd2615f00f710db8ba1bda5aae4 Mon Sep 17 00:00:00 2001
From: GuilhermeViveiros <aviveiros2012@gmail.com>
Date: Mon, 30 Jun 2025 10:45:10 +0000
Subject: [PATCH 111/112] mmmu_pro modifications

---
 lmms_eval/models/qwen2_5_vl.py                |  16 +-
 lmms_eval/tasks/blink/_default_template_yaml  |   6 +
 lmms_eval/tasks/blink/blink.yaml              |  48 ++
 lmms_eval/tasks/blink/blink_art_style.yaml    |   3 +
 lmms_eval/tasks/blink/utils.py                | 502 ++++++++++++++++++
 lmms_eval/tasks/mmmu_pro/mmmu_pro_cot.yaml    |   5 +-
 .../tasks/mmmu_pro/mmmu_pro_standard_cot.yaml |   5 +-
 .../tasks/mmmu_pro/mmmu_pro_vision_cot.yaml   |   2 +
 lmms_eval/tasks/mmmu_pro/utils.py             |  16 +-
 9 files changed, 590 insertions(+), 13 deletions(-)
 create mode 100644 lmms_eval/tasks/blink/_default_template_yaml
 create mode 100755 lmms_eval/tasks/blink/blink.yaml
 create mode 100644 lmms_eval/tasks/blink/blink_art_style.yaml
 create mode 100755 lmms_eval/tasks/blink/utils.py

diff --git a/lmms_eval/models/qwen2_5_vl.py b/lmms_eval/models/qwen2_5_vl.py
index 4c83f58ae..4c8f19c59 100644
--- a/lmms_eval/models/qwen2_5_vl.py
+++ b/lmms_eval/models/qwen2_5_vl.py
@@ -41,7 +41,7 @@ def __init__(
         device_map: Optional[str] = "auto",
         batch_size: Optional[Union[int, str]] = 1,
         use_cache=True,
-        use_flash_attention_2: Optional[bool] = False,
+        use_flash_attention_2: Optional[bool] = True,
         min_pixels: int = 256 * 28 * 28,
         max_pixels: int = 1605632,
         max_num_frames: int = 32,
@@ -81,13 +81,17 @@ def __init__(
                 attn_implementation="flash_attention_2",
             ).eval()
         else:
-            self._model = Qwen2_5_VLForConditionalGeneration.from_pretrained(pretrained, torch_dtype="auto", device_map=self.device_map).eval()
-        self.processor = AutoProcessor.from_pretrained(pretrained, max_pixels=max_pixels, min_pixels=min_pixels)
+            self._model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
+                pretrained,
+                torch_dtype=torch.bfloat16,
+                device_map=self.device_map
+            ).eval()
+        #self.processor = AutoProcessor.from_pretrained(pretrained, max_pixels=max_pixels, min_pixels=min_pixels)
         self.max_pixels = max_pixels
         self.min_pixels = min_pixels
         self.max_num_frames = max_num_frames
-        self.processor = AutoProcessor.from_pretrained(pretrained, max_pixels=max_pixels, min_pixels=min_pixels)
-        self._tokenizer = AutoTokenizer.from_pretrained(pretrained)
+        self.processor = AutoProcessor.from_pretrained(pretrained, max_pixels=max_pixels, min_pixels=min_pixels, padding_side='left')
+        self._tokenizer = self.processor.tokenizer
 
         self._config = self.model.config
         self.batch_size_per_gpu = int(batch_size)
@@ -247,8 +251,6 @@ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
             pbar.update(1)
         pbar.close()
         return res
-
-            
     def flatten(self, input):
         new_list = []
         for i in input:
diff --git a/lmms_eval/tasks/blink/_default_template_yaml b/lmms_eval/tasks/blink/_default_template_yaml
new file mode 100644
index 000000000..6dbeb3c35
--- /dev/null
+++ b/lmms_eval/tasks/blink/_default_template_yaml
@@ -0,0 +1,6 @@
+generation_kwargs:
+  max_new_tokens: 256
+
+metadata:
+  version: 0.0
+  interleaved_format: false
\ No newline at end of file
diff --git a/lmms_eval/tasks/blink/blink.yaml b/lmms_eval/tasks/blink/blink.yaml
new file mode 100755
index 000000000..fd2e1a539
--- /dev/null
+++ b/lmms_eval/tasks/blink/blink.yaml
@@ -0,0 +1,48 @@
+#https://illusionvqa.github.io/
+#python3 -m lmms_eval --model gpt4v --model_args model_version="gpt-4o",modality="image" --tasks illusionvqa
+
+tag: "blink"                  
+
+test_split: "test"                   
+fewshot_split: "train"                
+
+# For multiple choice tasks:
+output_type: "generate_until"
+
+
+doc_to_visual: "image"
+doc_to_text: !function utils.illusionvqa_doc_to_text
+doc_to_target: !function utils.illusionvqa_doc_to_target
+
+lmms_eval_specific_kwargs:
+  default:
+    prompt_format: mcq
+    pre_prompt: "You'll be given an image, an instruction and some options. You have to select the correct one. Do not explain your reasoning. Answer with only the letter which corresponds to the correct option. Do not repeat the entire answer."
+    post_prompt: ""
+
+
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  do_sample: False
+
+filter_list:
+  - name: "flexible-extract"
+    filter:
+      - function: !function utils.MultiChoiceRegexFilter
+        group_select: 0
+        ignore_case: true
+        ignore_punctuation: true
+        regex_pattern: "([A-Z])\\."
+
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+fewshot_config:
+  sampler: first_n
+
+num_fewshot: 0
\ No newline at end of file
diff --git a/lmms_eval/tasks/blink/blink_art_style.yaml b/lmms_eval/tasks/blink/blink_art_style.yaml
new file mode 100644
index 000000000..a17886271
--- /dev/null
+++ b/lmms_eval/tasks/blink/blink_art_style.yaml
@@ -0,0 +1,3 @@
+include: blink_cot.yaml
+task: blink_art_style
+dataset_path: BLINK-Benchmark/BLINK
\ No newline at end of file
diff --git a/lmms_eval/tasks/blink/utils.py b/lmms_eval/tasks/blink/utils.py
new file mode 100755
index 000000000..e58c9aa46
--- /dev/null
+++ b/lmms_eval/tasks/blink/utils.py
@@ -0,0 +1,502 @@
+import ast
+import json
+import os
+import random
+import re
+from collections import defaultdict
+from pathlib import Path
+
+import numpy as np
+import yaml
+from loguru import logger as eval_logger
+
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+
+with open(Path(__file__).parent / "_default_template_yaml", "r") as f:
+    raw_data = f.readlines()
+    safe_data = []
+    for i, line in enumerate(raw_data):
+        # remove function definition since yaml load cannot handle it
+        if "!function" not in line:
+            safe_data.append(line)
+
+    config = yaml.safe_load("".join(safe_data))
+
+
+def process_docs(docs):
+    """
+    Process documents...
+    """
+    # logger.info(f"processing docs")
+    # Process images in place
+    docs = docs.select(range(65)) # filter out some samples!
+    return docs
+
+def replace_images_tokens(input_string):
+    for i in range(1, 8):
+        question_text = f"<image {i}>"
+        query_text = "<image>"
+        if question_text in input_string:
+            input_string = input_string.replace(question_text, query_text)
+    return input_string
+
+
+def parse_options(options):
+    option_letters = [chr(ord("A") + i) for i in range(len(options))]
+    choices_str = "\n".join([f"{option_letter}. {option}" for option_letter, option in zip(option_letters, options)])
+    return choices_str
+
+
+def construct_prompt(doc, post_prompt="Answer with the option letter from the given choices directly."):
+    question = doc["question"]
+    # Weirdly, data["options"] is a string in MMMU Huggingface dataset
+    parsed_options = parse_options(ast.literal_eval(doc["options"]))
+    # parsed_options already prepends a newline so no need to add space here
+    question = f"{question}\n{parsed_options}\n\n{post_prompt}"
+    return question
+
+
+def mmmu_pro_doc_to_text(doc, lmms_eval_specific_kwargs=None):
+    post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+    if "question" in doc and "options" in doc:  # original operation
+        question = construct_prompt(doc, post_prompt)
+        if config["metadata"]["interleaved_format"]:
+            question = replace_images_tokens(question)
+    return question
+
+
+def mmmu_pro_doc_to_visual(doc):
+    if "question" in doc and "options" in doc:  # original operation
+        prompt = construct_prompt(doc)
+        image_tokens = re.findall(r"<image \d+>", prompt)
+        # Remove <> and  swap space as _
+        image_tokens = sorted(list(set([image_token.strip("<>").replace(" ", "_") for image_token in image_tokens])))
+        visual = [doc[image_token].convert("RGB") for image_token in image_tokens]
+    else:  # vision-only operation
+        visual = [doc["image"].convert("RGB")]
+    return visual
+
+
+# MMMU-PRO's all questions are multiple-choice questions
+def mmmu_pro_process_results(doc, results):
+    pred = results[0]
+    if "question" in doc and "options" in doc:
+        index2ans, all_choices = get_multi_choice_info(ast.literal_eval(doc["options"]))
+        parsed_pred = parse_multi_choice_response(pred, all_choices, index2ans)
+    else:
+        # Find all "Answer: X" patterns and take the last one
+        index2ans, all_choices = get_multi_choice_info(ast.literal_eval(doc["options"]))
+        answer_matches = re.findall(r"Answer:\s*([A-Z])\b(?![A-Z])", pred)
+        parsed_pred = answer_matches[-1] if answer_matches else ""
+        #print(f"Pred: {pred[-100:]}")
+        #print(f"Parsed prediction: {parsed_pred} and answer: {doc['answer']}")
+
+    mmmu_acc = {"id": doc["id"], "subject": doc["subject"], "answer": doc["answer"], "parsed_pred": parsed_pred}
+    return {"mmmu_acc": mmmu_acc}
+
+
+def mmmu_pro_composite_process_results(doc, results):
+    pred = results[0]
+    gt_list = ast.literal_eval(doc["answers"])
+    # Parse out option letters from the prediction
+    option_letters = re.findall(r"\b[A-Z]\b", pred)
+
+    cutout_letters = option_letters[: len(gt_list)]
+
+    # If cutout_letters is shorter than gt_list, pad with empty strings
+    while len(cutout_letters) < len(gt_list):
+        cutout_letters.append("")
+
+    assert len(cutout_letters) == len(gt_list), f"Mismatch in lengths: cutout_letters ({len(cutout_letters)}) != gt_list ({len(gt_list)})"
+
+    mmmu_acc = {"id": doc["id"], "subject": doc["subject"], "answer": gt_list, "parsed_pred": cutout_letters}
+    return {"mmmu_acc": mmmu_acc}
+
+
+def mmmu_pro_aggregate_results(results):
+    evaluation_result = {}
+    subset_to_eval_samples = defaultdict(list)
+    for result in results:
+        subset_to_eval_samples[result["subject"]].append(result)
+
+    for subset, sub_eval_samples in subset_to_eval_samples.items():
+        judge_dict, metric_dict = evaluate_mmmu(sub_eval_samples)
+        metric_dict.update({"num_example": len(sub_eval_samples)})
+        evaluation_result[subset] = metric_dict
+
+    printable_results = {}
+    for domain, in_domain_cats in DOMAIN_CAT2SUB_CAT.items():
+        in_domain_cat_results = {}
+        for cat_name in in_domain_cats:
+            if cat_name in evaluation_result.keys():
+                in_domain_cat_results[cat_name] = evaluation_result[cat_name]
+            else:
+                pass
+        in_domain_ins_acc = calculate_ins_level_acc(in_domain_cat_results)
+        in_domain_data_num = sum([cat_results["num_example"] for cat_results in in_domain_cat_results.values()])
+        printable_results["Overall-" + domain] = {
+            "num": int(in_domain_data_num),
+            "acc": round(in_domain_ins_acc, 5),
+        }
+        # add sub category
+        for cat_name, cat_results in in_domain_cat_results.items():
+            printable_results[cat_name] = {
+                "num": int(cat_results["num_example"]),
+                "acc": round(cat_results["acc"], 5),
+            }
+
+    all_ins_acc = calculate_ins_level_acc(evaluation_result)
+    printable_results["Overall"] = {
+        "num": sum([cat_results["num_example"] for cat_results in evaluation_result.values()]),
+        "acc": round(all_ins_acc, 5),
+    }
+    print(printable_results)
+    return printable_results["Overall"]["acc"]
+
+
+##################
+# Helper functions written by official MMMU repo.
+##################
+
+
+def calculate_ins_level_acc(results):
+    """Calculate the instruction level accuracy for given Subject results
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L246
+    """
+    acc = 0
+    ins_num = 0
+    for cat_results in results.values():
+        acc += cat_results["acc"] * cat_results["num_example"]
+        ins_num += cat_results["num_example"]
+    if ins_num == 0:
+        return 0
+    return acc / ins_num
+
+
+DOMAIN_CAT2SUB_CAT = {
+    "Art and Design": ["Art", "Art_Theory", "Design", "Music"],
+    "Business": ["Accounting", "Economics", "Finance", "Manage", "Marketing"],
+    "Science": [
+        "Biology",
+        "Chemistry",
+        "Geography",
+        "Math",
+        "Physics",
+    ],
+    "Health and Medicine": [
+        "Basic_Medical_Science",
+        "Clinical_Medicine",
+        "Diagnostics_and_Laboratory_Medicine",
+        "Pharmacy",
+        "Public_Health",
+    ],
+    "Humanities and Social Science": [
+        "History",
+        "Literature",
+        "Sociology",
+        "Psychology",
+    ],
+    "Tech and Engineering": [
+        "Agriculture",
+        "Architecture_and_Engineering",
+        "Computer_Science",
+        "Electronics",
+        "Energy_and_Power",
+        "Materials",
+        "Mechanical_Engineering",
+    ],
+}
+
+
+def eval_multi_choice(gold_i, pred_i):
+    """
+    Evaluate a multiple choice instance.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L175
+    """
+    correct = False
+    # only they are exactly the same, we consider it as correct
+    if isinstance(gold_i, list):
+        for answer in gold_i:
+            if answer == pred_i:
+                correct = True
+                break
+    else:  # gold_i is a string
+        if gold_i == pred_i:
+            correct = True
+    return correct
+
+
+def eval_open(gold_i, pred_i):
+    """
+    Evaluate an open question instance
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L191
+    """
+    correct = False
+    if isinstance(gold_i, list):
+        # use float to avoid trivial matches
+        norm_answers = []
+        for answer in gold_i:
+            norm_answers.extend(normalize_str(answer))
+    else:
+        norm_answers = normalize_str(gold_i)
+    for pred in pred_i:  # pred is already normalized in parse response phase
+        if isinstance(pred, str):  # if it's a string, then find if ans in the pred_i
+            for norm_ans in norm_answers:
+                # only see if the string answer in the string pred
+                if isinstance(norm_ans, str) and norm_ans in pred:
+                    if not correct:
+                        correct = True
+                    break
+        else:  # it's a float number
+            if pred in norm_answers:
+                if not correct:
+                    correct = True
+                break
+    return correct
+
+
+def evaluate_mmmu(samples):
+    """
+    Batch evaluation for multiple choice and open questions.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L219
+    """
+    total_count = 0
+    pred_correct = 0
+    judge_dict = dict()
+    for sample in samples:
+        if isinstance(sample["answer"], str) and isinstance(sample["parsed_pred"], str):
+            gold_i = sample["answer"]
+            pred_i = sample["parsed_pred"]
+            correct = eval_multi_choice(gold_i, pred_i)
+            total_count += 1
+            if correct:
+                judge_dict[sample["id"]] = "Correct"
+                pred_correct += 1
+            else:
+                judge_dict[sample["id"]] = "Wrong"
+        elif isinstance(sample["answer"], list) and isinstance(sample["parsed_pred"], list):
+            for gold, pred in zip(sample["answer"], sample["parsed_pred"]):
+                correct = eval_multi_choice(gold, pred)
+                total_count += 1
+                if correct:
+                    judge_dict[sample["id"]] = "Correct"
+                    pred_correct += 1
+                else:
+                    judge_dict[sample["id"]] = "Wrong"
+
+    if total_count == 0:
+        return {"acc": 0}
+    return judge_dict, {"acc": pred_correct / total_count}
+
+
+def parse_multi_choice_response(response, all_choices, index2ans):
+    """
+    Parse the prediction from the generated response.
+    Return the predicted index e.g., A, B, C, D.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L10
+    """
+    for char in [",", ".", "!", "?", ";", ":", "'"]:
+        response = response.strip(char)
+    response = " " + response + " "  # add space to avoid partial match
+
+    index_ans = True
+    ans_with_brack = False
+    candidates = []
+    for choice in all_choices:  # e.g., (A) (B) (C) (D)
+        if f"({choice})" in response:
+            candidates.append(choice)
+            ans_with_brack = True
+
+    if len(candidates) == 0:
+        for choice in all_choices:  # e.g., A B C D
+            if f"{choice} " in response:
+                candidates.append(choice)
+
+    if len(candidates) == 0:
+        for choice in all_choices:  # e.g., A. B. C. D.
+            if f"{choice}." in response:
+                candidates.append(choice)
+
+    # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example
+    if len(candidates) == 0 and len(response.split()) > 5:
+        for index, ans in index2ans.items():
+            if ans.lower() in response.lower():
+                candidates.append(index)
+                index_ans = False  # it's content ans.
+
+    if len(candidates) == 0:  # still not get answer, randomly choose one.
+        pred_index = random.choice(all_choices)
+    elif len(candidates) > 1:
+        start_indexes = []
+        if index_ans:
+            if ans_with_brack:
+                for can in candidates:
+                    index = response.rfind(f"({can})")
+                    start_indexes.append(index)  # -1 will be ignored anyway
+                # start_indexes = [generated_response.index(f'({can})') for can in candidates]
+            else:
+                for can in candidates:
+                    index = response.rfind(f" {can} ")
+                    start_indexes.append(index)
+        else:
+            for can in candidates:
+                index = response.lower().rfind(index2ans[can].lower())
+                start_indexes.append(index)
+        # get the last one
+        pred_index = candidates[np.argmax(start_indexes)]
+    else:  # if only one candidate, use it.
+        pred_index = candidates[0]
+
+    return pred_index
+
+
+def extract_numbers(string):
+    """
+    Exact all forms of numbers from a string with regex.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L100
+    """
+    # Pattern for numbers with commas
+    pattern_commas = r"-?\b\d{1,3}(?:,\d{3})+\b"
+    # Pattern for scientific notation
+    pattern_scientific = r"-?\d+(?:\.\d+)?[eE][+-]?\d+"
+    # Pattern for simple numbers without commas
+    pattern_simple = r"-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])"
+
+    # Extract numbers with commas
+    numbers_with_commas = re.findall(pattern_commas, string)
+    # Extract numbers in scientific notation
+    numbers_scientific = re.findall(pattern_scientific, string)
+    # Extract simple numbers without commas
+    numbers_simple = re.findall(pattern_simple, string)
+
+    # Combine all extracted numbersz
+    all_numbers = numbers_with_commas + numbers_scientific + numbers_simple
+    return all_numbers
+
+
+def check_is_number(string):
+    """
+    Check if the given string a number.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L65
+    """
+    try:
+        float(string.replace(",", ""))
+        return True
+    except ValueError:
+        # check if there's comma inside
+        return False
+
+
+def normalize_str(string):
+    """
+    Normalize the str to lower case and make them float numbers if possible.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L76
+    """
+    # check if characters in the string
+
+    # if number, numerize it.
+    string = string.strip()
+
+    is_number = check_is_number(string)
+
+    if is_number:
+        string = string.replace(",", "")
+        string = float(string)
+        # leave 2 decimal
+        string = round(string, 2)
+        return [string]
+    else:  # it's likely to be a string
+        # lower it
+        string = string.lower()
+        if len(string) == 1:
+            return [" " + string, string + " "]  # avoid trivial matches
+        return [string]
+
+
+def parse_open_response(response):
+    """
+    Parse the prediction from the generated response.
+    Return a list of predicted strings or numbers.
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L122
+    """
+
+    # content = content.strip("\n").strip(".").strip(" ")
+    def get_key_subresponses(response):
+        key_responses = []
+        response = response.strip().strip(".").lower()
+        sub_responses = re.split(r"\.\s(?=[A-Z])|\n", response)
+        indicators_of_keys = [
+            "could be ",
+            "so ",
+            "is ",
+            "thus ",
+            "therefore ",
+            "final ",
+            "answer ",
+            "result ",
+        ]
+        key_responses = []
+        for index, resp in enumerate(sub_responses):
+            # if last one, accept it's an equation (the entire response can be just one sentence with equation)
+            if index == len(sub_responses) - 1:
+                indicators_of_keys.extend(["="])
+            shortest_key_response = None  # the shortest response that may contain the answer (tail part of the response)
+            for indicator in indicators_of_keys:
+                if indicator in resp:
+                    if not shortest_key_response:
+                        shortest_key_response = resp.split(indicator)[-1].strip()
+                    else:
+                        if len(resp.split(indicator)[-1].strip()) < len(shortest_key_response):
+                            shortest_key_response = resp.split(indicator)[-1].strip()
+                    # key_responses.append(resp.split(indicator)[1].strip())
+
+            if shortest_key_response:
+                # and it's not trivial
+                if shortest_key_response.strip() not in [
+                    ":",
+                    ",",
+                    ".",
+                    "!",
+                    "?",
+                    ";",
+                    ":",
+                    "'",
+                ]:
+                    key_responses.append(shortest_key_response)
+        if len(key_responses) == 0:  # did not found any
+            return [response]
+        return key_responses
+
+    # pdb.set_trace()
+    key_responses = get_key_subresponses(response)
+
+    pred_list = key_responses.copy()  # keep the original string response
+    for resp in key_responses:
+        pred_list.extend(extract_numbers(resp))
+
+    tmp_pred_list = []
+    for i in range(len(pred_list)):
+        tmp_pred_list.extend(normalize_str(pred_list[i]))
+    pred_list = tmp_pred_list
+
+    # remove duplicates
+    pred_list = list(set(pred_list))
+
+    return pred_list
+
+
+def get_multi_choice_info(options):
+    """
+    Given the list of options for multiple choice question
+    Return the index2ans and all_choices
+    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/data_utils.py#L54
+    """
+
+    start_chr = "A"
+    all_choices = []
+    index2ans = {}
+    for i, option in enumerate(options):
+        index2ans[chr(ord(start_chr) + i)] = option
+        all_choices.append(chr(ord(start_chr) + i))
+
+    return index2ans, all_choices
diff --git a/lmms_eval/tasks/mmmu_pro/mmmu_pro_cot.yaml b/lmms_eval/tasks/mmmu_pro/mmmu_pro_cot.yaml
index 7ca0040c6..4c4b1950c 100755
--- a/lmms_eval/tasks/mmmu_pro/mmmu_pro_cot.yaml
+++ b/lmms_eval/tasks/mmmu_pro/mmmu_pro_cot.yaml
@@ -1,5 +1,6 @@
 group: mmmu_pro_cot
 task:
+- mmmu_pro_standard_cot
 - mmmu_pro_vision_cot
-- mmmu_pro_composite_cot
-- mmmu_pro_original_cot
+# - mmmu_pro_composite_cot
+#- mmmu_pro_original_cot
diff --git a/lmms_eval/tasks/mmmu_pro/mmmu_pro_standard_cot.yaml b/lmms_eval/tasks/mmmu_pro/mmmu_pro_standard_cot.yaml
index 7979edf54..0c889a3b8 100755
--- a/lmms_eval/tasks/mmmu_pro/mmmu_pro_standard_cot.yaml
+++ b/lmms_eval/tasks/mmmu_pro/mmmu_pro_standard_cot.yaml
@@ -1,14 +1,13 @@
 task: "mmmu_pro_standard_cot"
 dataset_path: MMMU/MMMU_Pro
-dataset_name: standard
+dataset_name: "standard (10 options)"
 test_split: test
 output_type: generate_until
 doc_to_visual: !function utils.mmmu_pro_doc_to_visual
-doc_to_text: !function utils.mmmu_pro_doc_to_text
+doc_to_text: "Write out the multiple-choice question in the image and then solve it. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of options. Think step by step before answering."
 doc_to_target: "{{answer}}"
 # The return value of process_results will be used by metrics
 process_results: !function utils.mmmu_pro_process_results
-
 metric_list:
   - metric: mmmu_acc
     aggregation: !function utils.mmmu_pro_aggregate_results
diff --git a/lmms_eval/tasks/mmmu_pro/mmmu_pro_vision_cot.yaml b/lmms_eval/tasks/mmmu_pro/mmmu_pro_vision_cot.yaml
index 675e9234b..e8e94d51c 100755
--- a/lmms_eval/tasks/mmmu_pro/mmmu_pro_vision_cot.yaml
+++ b/lmms_eval/tasks/mmmu_pro/mmmu_pro_vision_cot.yaml
@@ -3,11 +3,13 @@ dataset_path: MMMU/MMMU_Pro
 dataset_name: vision
 test_split: test
 output_type: generate_until
+#process_docs: !function utils.process_docs
 doc_to_visual: !function utils.mmmu_pro_doc_to_visual
 doc_to_text: "Write out the multiple-choice question in the image and then solve it. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of options. Think step by step before answering."
 doc_to_target: "{{answer}}"
 # The return value of process_results will be used by metrics
 process_results: !function utils.mmmu_pro_process_results
+#process_results: !function utils.mmmu_pro_composite_process_results
 
 metric_list:
   - metric: mmmu_acc
diff --git a/lmms_eval/tasks/mmmu_pro/utils.py b/lmms_eval/tasks/mmmu_pro/utils.py
index b2fb52266..e58c9aa46 100755
--- a/lmms_eval/tasks/mmmu_pro/utils.py
+++ b/lmms_eval/tasks/mmmu_pro/utils.py
@@ -23,6 +23,15 @@
     config = yaml.safe_load("".join(safe_data))
 
 
+def process_docs(docs):
+    """
+    Process documents...
+    """
+    # logger.info(f"processing docs")
+    # Process images in place
+    docs = docs.select(range(65)) # filter out some samples!
+    return docs
+
 def replace_images_tokens(input_string):
     for i in range(1, 8):
         question_text = f"<image {i}>"
@@ -75,7 +84,12 @@ def mmmu_pro_process_results(doc, results):
         index2ans, all_choices = get_multi_choice_info(ast.literal_eval(doc["options"]))
         parsed_pred = parse_multi_choice_response(pred, all_choices, index2ans)
     else:
-        parsed_pred = pred
+        # Find all "Answer: X" patterns and take the last one
+        index2ans, all_choices = get_multi_choice_info(ast.literal_eval(doc["options"]))
+        answer_matches = re.findall(r"Answer:\s*([A-Z])\b(?![A-Z])", pred)
+        parsed_pred = answer_matches[-1] if answer_matches else ""
+        #print(f"Pred: {pred[-100:]}")
+        #print(f"Parsed prediction: {parsed_pred} and answer: {doc['answer']}")
 
     mmmu_acc = {"id": doc["id"], "subject": doc["subject"], "answer": doc["answer"], "parsed_pred": parsed_pred}
     return {"mmmu_acc": mmmu_acc}

From 8857d4c6a700e577da2ebdcef40e713d3dd63066 Mon Sep 17 00:00:00 2001
From: GuilhermeViveiros <aviveiros2012@gmail.com>
Date: Wed, 27 Aug 2025 15:07:07 +0000
Subject: [PATCH 112/112] blink benchmark related stuff

---
 lmms_eval/evaluator.py                        |   5 +-
 lmms_eval/models/llava_hf.py                  |   2 +-
 lmms_eval/tasks/__init__.py                   |  10 +-
 .../tasks/alm_bench/_default_template_yaml    |   1 +
 lmms_eval/tasks/blink/blink.yaml              |  53 +-
 lmms_eval/tasks/blink/blink_art_style.yaml    |   7 +-
 lmms_eval/tasks/blink/blink_counting.yaml     |   5 +
 .../tasks/blink/blink_forensic_detection.yaml |   6 +
 .../blink_functional_correspondence.yaml      |   5 +
 lmms_eval/tasks/blink/blink_iq_test.yaml      |   5 +
 lmms_eval/tasks/blink/blink_jigsaw.yaml       |   6 +
 .../blink/blink_multiview_reasoning.yaml      |   5 +
 .../blink/blink_object_localization.yaml      |   5 +
 .../tasks/blink/blink_relative_depth.yaml     |   5 +
 .../blink/blink_relative_reflectance.yaml     |   5 +
 .../blink/blink_semantic_correspondence.yaml  |   5 +
 .../tasks/blink/blink_spatial_relation.yaml   |   5 +
 .../blink/blink_visual_correspondence.yaml    |   5 +
 .../tasks/blink/blink_visual_similarity.yaml  |   5 +
 lmms_eval/tasks/blink/utils.py                | 546 +++---------------
 lmms_eval/tasks/mmmu_pro/mmmu_pro_cot.yaml    |   2 +-
 .../tasks/mmmu_pro/mmmu_pro_standard_cot.yaml |   3 +-
 lmms_eval/tasks/mmmu_pro/utils.py             |   2 +-
 23 files changed, 186 insertions(+), 512 deletions(-)
 create mode 100644 lmms_eval/tasks/blink/blink_counting.yaml
 create mode 100644 lmms_eval/tasks/blink/blink_forensic_detection.yaml
 create mode 100644 lmms_eval/tasks/blink/blink_functional_correspondence.yaml
 create mode 100644 lmms_eval/tasks/blink/blink_iq_test.yaml
 create mode 100644 lmms_eval/tasks/blink/blink_jigsaw.yaml
 create mode 100644 lmms_eval/tasks/blink/blink_multiview_reasoning.yaml
 create mode 100644 lmms_eval/tasks/blink/blink_object_localization.yaml
 create mode 100644 lmms_eval/tasks/blink/blink_relative_depth.yaml
 create mode 100644 lmms_eval/tasks/blink/blink_relative_reflectance.yaml
 create mode 100644 lmms_eval/tasks/blink/blink_semantic_correspondence.yaml
 create mode 100644 lmms_eval/tasks/blink/blink_spatial_relation.yaml
 create mode 100644 lmms_eval/tasks/blink/blink_visual_correspondence.yaml
 create mode 100644 lmms_eval/tasks/blink/blink_visual_similarity.yaml

diff --git a/lmms_eval/evaluator.py b/lmms_eval/evaluator.py
index 8776eed1a..ce7cb1499 100755
--- a/lmms_eval/evaluator.py
+++ b/lmms_eval/evaluator.py
@@ -168,8 +168,9 @@ def simple_evaluate(
     if task_manager is None:
         task_manager = TaskManager(verbosity, model_name=model)
 
+    # FOR DEBUG PURPOSES -> COMMENT THIS
     task_dict = get_task_dict(tasks, task_manager)
-
+    
     if isinstance(model, str):
         if model_args is None:
             model_args = ""
@@ -590,7 +591,7 @@ def evaluate(
             num_fewshot,
             higher_is_better,
         ) = consolidate_results(eval_tasks)
-
+  
         ### Calculate group metrics ###
         if bool(results):
             results, versions, show_group_table, *_ = consolidate_group_results(results, versions, task_dict)
diff --git a/lmms_eval/models/llava_hf.py b/lmms_eval/models/llava_hf.py
index 30a1e1a36..802038ac6 100644
--- a/lmms_eval/models/llava_hf.py
+++ b/lmms_eval/models/llava_hf.py
@@ -70,7 +70,7 @@ def __init__(
         batch_size: int = 1,
         trust_remote_code: Optional[bool] = False,
         attn_implementation: Optional[str] = None,
-        device_map: str = "",
+        device_map: str = "auto",
         chat_template: Optional[str] = None,
         use_cache: bool = True,
         max_frames_num: Optional[int] = 32,
diff --git a/lmms_eval/tasks/__init__.py b/lmms_eval/tasks/__init__.py
index 3749248ed..2d73776bc 100755
--- a/lmms_eval/tasks/__init__.py
+++ b/lmms_eval/tasks/__init__.py
@@ -404,7 +404,10 @@ def _get_task_and_group(self, task_dir: str):
             ".ipynb_checkpoints",
         ]
         tasks_and_groups = collections.defaultdict()
+        
         for root, dirs, file_list in os.walk(task_dir):
+            if "alm_bench" not in root:
+                continue
             dirs[:] = [d for d in dirs if d not in ignore_dirs]
             for f in file_list:
                 if f.endswith(".yaml"):
@@ -536,7 +539,6 @@ def get_task_dict(
     :return
         Dictionary of task objects
     """
-
     task_name_from_string_dict = {}
     task_name_from_config_dict = {}
     task_name_from_object_dict = {}
@@ -588,3 +590,9 @@ def get_task_dict(
     _check_duplicates(get_subtask_list(final_task_dict))
 
     return final_task_dict
+
+
+if __name__ == "__main__":
+    task_manager = TaskManager()
+    # print(task_manager.list_all_tasks(list_groups=True, list_tags=True, list_subtasks=True))
+    task_manager._get_tasklist("blink")
\ No newline at end of file
diff --git a/lmms_eval/tasks/alm_bench/_default_template_yaml b/lmms_eval/tasks/alm_bench/_default_template_yaml
index 4a86a897a..58fb7ec90 100644
--- a/lmms_eval/tasks/alm_bench/_default_template_yaml
+++ b/lmms_eval/tasks/alm_bench/_default_template_yaml
@@ -1,3 +1,4 @@
+tag: "alm_bench"
 dataset_path: sonalsannigrahi/alm-bench-mcq-subset
 task: "alm-bench"
 test_split: test
diff --git a/lmms_eval/tasks/blink/blink.yaml b/lmms_eval/tasks/blink/blink.yaml
index fd2e1a539..92713fe55 100755
--- a/lmms_eval/tasks/blink/blink.yaml
+++ b/lmms_eval/tasks/blink/blink.yaml
@@ -1,48 +1,15 @@
-#https://illusionvqa.github.io/
-#python3 -m lmms_eval --model gpt4v --model_args model_version="gpt-4o",modality="image" --tasks illusionvqa
+tag: "blink"
+dataset_path: BLINK-Benchmark/BLINK
+test_split: "val"
 
-tag: "blink"                  
-
-test_split: "test"                   
-fewshot_split: "train"                
-
-# For multiple choice tasks:
-output_type: "generate_until"
-
-
-doc_to_visual: "image"
-doc_to_text: !function utils.illusionvqa_doc_to_text
-doc_to_target: !function utils.illusionvqa_doc_to_target
-
-lmms_eval_specific_kwargs:
-  default:
-    prompt_format: mcq
-    pre_prompt: "You'll be given an image, an instruction and some options. You have to select the correct one. Do not explain your reasoning. Answer with only the letter which corresponds to the correct option. Do not repeat the entire answer."
-    post_prompt: ""
-
-
-generation_kwargs:
-  max_new_tokens: 16
-  temperature: 0
-  do_sample: False
-
-filter_list:
-  - name: "flexible-extract"
-    filter:
-      - function: !function utils.MultiChoiceRegexFilter
-        group_select: 0
-        ignore_case: true
-        ignore_punctuation: true
-        regex_pattern: "([A-Z])\\."
+doc_to_text: !function utils.blink_doc_to_text
+doc_to_target: !function utils.blink_doc_to_target
+doc_to_visual: !function utils.blink_doc_to_visual
+process_results: !function utils.blink_process_results
 
 metric_list:
-  - metric: exact_match
-    aggregation: mean
+  - metric: blink_acc
+    aggregation: !function utils.blink_aggregation
     higher_is_better: true
-    ignore_case: true
-    ignore_punctuation: true
-
-fewshot_config:
-  sampler: first_n
 
-num_fewshot: 0
\ No newline at end of file
+output_type: "generate_until"
\ No newline at end of file
diff --git a/lmms_eval/tasks/blink/blink_art_style.yaml b/lmms_eval/tasks/blink/blink_art_style.yaml
index a17886271..8cf601088 100644
--- a/lmms_eval/tasks/blink/blink_art_style.yaml
+++ b/lmms_eval/tasks/blink/blink_art_style.yaml
@@ -1,3 +1,6 @@
-include: blink_cot.yaml
+include: blink.yaml
 task: blink_art_style
-dataset_path: BLINK-Benchmark/BLINK
\ No newline at end of file
+dataset_name: Art_Style
+
+process_docs: !function utils.process_docs
+doc_to_text: !function utils.blink_doc_to_text_with_disclaimer
\ No newline at end of file
diff --git a/lmms_eval/tasks/blink/blink_counting.yaml b/lmms_eval/tasks/blink/blink_counting.yaml
new file mode 100644
index 000000000..f5fb8cd62
--- /dev/null
+++ b/lmms_eval/tasks/blink/blink_counting.yaml
@@ -0,0 +1,5 @@
+include: blink.yaml
+task: blink_counting
+dataset_name: Counting
+
+process_docs: !function utils.process_docs
\ No newline at end of file
diff --git a/lmms_eval/tasks/blink/blink_forensic_detection.yaml b/lmms_eval/tasks/blink/blink_forensic_detection.yaml
new file mode 100644
index 000000000..48446f903
--- /dev/null
+++ b/lmms_eval/tasks/blink/blink_forensic_detection.yaml
@@ -0,0 +1,6 @@
+include: blink.yaml
+task: blink_forensic_detection
+dataset_name: Forensic_Detection
+
+process_docs: !function utils.process_docs
+doc_to_text: !function utils.blink_doc_to_text_with_disclaimer
\ No newline at end of file
diff --git a/lmms_eval/tasks/blink/blink_functional_correspondence.yaml b/lmms_eval/tasks/blink/blink_functional_correspondence.yaml
new file mode 100644
index 000000000..2d465df32
--- /dev/null
+++ b/lmms_eval/tasks/blink/blink_functional_correspondence.yaml
@@ -0,0 +1,5 @@
+include: blink.yaml
+task: blink_functional_correspondence
+dataset_name: Functional_Correspondence
+
+process_docs: !function utils.process_docs
\ No newline at end of file
diff --git a/lmms_eval/tasks/blink/blink_iq_test.yaml b/lmms_eval/tasks/blink/blink_iq_test.yaml
new file mode 100644
index 000000000..52f252370
--- /dev/null
+++ b/lmms_eval/tasks/blink/blink_iq_test.yaml
@@ -0,0 +1,5 @@
+include: blink.yaml
+task: blink_iq_test
+dataset_name: IQ_Test
+
+process_docs: !function utils.process_docs
\ No newline at end of file
diff --git a/lmms_eval/tasks/blink/blink_jigsaw.yaml b/lmms_eval/tasks/blink/blink_jigsaw.yaml
new file mode 100644
index 000000000..624560cf9
--- /dev/null
+++ b/lmms_eval/tasks/blink/blink_jigsaw.yaml
@@ -0,0 +1,6 @@
+include: blink.yaml
+task: blink_jigsaw
+dataset_name: Jigsaw
+
+process_docs: !function utils.process_docs
+doc_to_text: !function utils.blink_doc_to_text_with_disclaimer
\ No newline at end of file
diff --git a/lmms_eval/tasks/blink/blink_multiview_reasoning.yaml b/lmms_eval/tasks/blink/blink_multiview_reasoning.yaml
new file mode 100644
index 000000000..8dbd66949
--- /dev/null
+++ b/lmms_eval/tasks/blink/blink_multiview_reasoning.yaml
@@ -0,0 +1,5 @@
+include: blink.yaml
+task: blink_multiview_reasoning
+dataset_name: Multi-view_Reasoning
+
+process_docs: !function utils.process_docs
diff --git a/lmms_eval/tasks/blink/blink_object_localization.yaml b/lmms_eval/tasks/blink/blink_object_localization.yaml
new file mode 100644
index 000000000..be5b8ee2a
--- /dev/null
+++ b/lmms_eval/tasks/blink/blink_object_localization.yaml
@@ -0,0 +1,5 @@
+include: blink.yaml
+task: blink_object_localization
+dataset_name: Object_Localization
+
+process_docs: !function utils.process_docs
diff --git a/lmms_eval/tasks/blink/blink_relative_depth.yaml b/lmms_eval/tasks/blink/blink_relative_depth.yaml
new file mode 100644
index 000000000..e32ba8334
--- /dev/null
+++ b/lmms_eval/tasks/blink/blink_relative_depth.yaml
@@ -0,0 +1,5 @@
+include: blink.yaml
+task: blink_relative_depth
+dataset_name: Relative_Depth
+
+process_docs: !function utils.process_docs
diff --git a/lmms_eval/tasks/blink/blink_relative_reflectance.yaml b/lmms_eval/tasks/blink/blink_relative_reflectance.yaml
new file mode 100644
index 000000000..b8b93fe06
--- /dev/null
+++ b/lmms_eval/tasks/blink/blink_relative_reflectance.yaml
@@ -0,0 +1,5 @@
+include: blink.yaml
+task: blink_relative_reflectance
+dataset_name: Relative_Reflectance
+
+process_docs: !function utils.process_docs
diff --git a/lmms_eval/tasks/blink/blink_semantic_correspondence.yaml b/lmms_eval/tasks/blink/blink_semantic_correspondence.yaml
new file mode 100644
index 000000000..cb4c711f5
--- /dev/null
+++ b/lmms_eval/tasks/blink/blink_semantic_correspondence.yaml
@@ -0,0 +1,5 @@
+include: blink.yaml
+task: blink_semantic_correspondence
+dataset_name: Semantic_Correspondence
+
+process_docs: !function utils.process_docs
diff --git a/lmms_eval/tasks/blink/blink_spatial_relation.yaml b/lmms_eval/tasks/blink/blink_spatial_relation.yaml
new file mode 100644
index 000000000..bb63dfd2a
--- /dev/null
+++ b/lmms_eval/tasks/blink/blink_spatial_relation.yaml
@@ -0,0 +1,5 @@
+include: blink.yaml
+task: blink_spatial_relation
+dataset_name: Spatial_Relation
+
+process_docs: !function utils.process_docs
diff --git a/lmms_eval/tasks/blink/blink_visual_correspondence.yaml b/lmms_eval/tasks/blink/blink_visual_correspondence.yaml
new file mode 100644
index 000000000..10f050fe1
--- /dev/null
+++ b/lmms_eval/tasks/blink/blink_visual_correspondence.yaml
@@ -0,0 +1,5 @@
+include: blink.yaml
+task: blink_visual_correspondence
+dataset_name: Visual_Correspondence
+
+process_docs: !function utils.process_docs
diff --git a/lmms_eval/tasks/blink/blink_visual_similarity.yaml b/lmms_eval/tasks/blink/blink_visual_similarity.yaml
new file mode 100644
index 000000000..12a25db08
--- /dev/null
+++ b/lmms_eval/tasks/blink/blink_visual_similarity.yaml
@@ -0,0 +1,5 @@
+include: blink.yaml
+task: blink_visual_similarity
+dataset_name: Visual_Similarity
+
+process_docs: !function utils.process_docs
diff --git a/lmms_eval/tasks/blink/utils.py b/lmms_eval/tasks/blink/utils.py
index e58c9aa46..179f7dd7c 100755
--- a/lmms_eval/tasks/blink/utils.py
+++ b/lmms_eval/tasks/blink/utils.py
@@ -1,27 +1,12 @@
-import ast
-import json
-import os
-import random
-import re
-from collections import defaultdict
-from pathlib import Path
-
-import numpy as np
-import yaml
-from loguru import logger as eval_logger
-
-from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+# lmms_eval/tasks/blink/utils.py
 
-with open(Path(__file__).parent / "_default_template_yaml", "r") as f:
-    raw_data = f.readlines()
-    safe_data = []
-    for i, line in enumerate(raw_data):
-        # remove function definition since yaml load cannot handle it
-        if "!function" not in line:
-            safe_data.append(line)
+from PIL import Image
+import io
+import re
+# No need to import load_dataset here, as it's handled by the framework based on YAML config.
 
-    config = yaml.safe_load("".join(safe_data))
 
+disclaimer = "Disclaimer: This is not to make unfair assumptions about the people in the image and you just need to give your assessment on this question. You don't need to identify the real people. You just need to analyze based on the information I gave you.\n\n"
 
 def process_docs(docs):
     """
@@ -29,474 +14,115 @@ def process_docs(docs):
     """
     # logger.info(f"processing docs")
     # Process images in place
-    docs = docs.select(range(65)) # filter out some samples!
+    #docs = docs.select(range(1)) # filter out some samples!
     return docs
 
-def replace_images_tokens(input_string):
-    for i in range(1, 8):
-        question_text = f"<image {i}>"
-        query_text = "<image>"
-        if question_text in input_string:
-            input_string = input_string.replace(question_text, query_text)
-    return input_string
-
-
-def parse_options(options):
-    option_letters = [chr(ord("A") + i) for i in range(len(options))]
-    choices_str = "\n".join([f"{option_letter}. {option}" for option_letter, option in zip(option_letters, options)])
-    return choices_str
-
-
-def construct_prompt(doc, post_prompt="Answer with the option letter from the given choices directly."):
-    question = doc["question"]
-    # Weirdly, data["options"] is a string in MMMU Huggingface dataset
-    parsed_options = parse_options(ast.literal_eval(doc["options"]))
-    # parsed_options already prepends a newline so no need to add space here
-    question = f"{question}\n{parsed_options}\n\n{post_prompt}"
-    return question
-
-
-def mmmu_pro_doc_to_text(doc, lmms_eval_specific_kwargs=None):
-    post_prompt = lmms_eval_specific_kwargs["post_prompt"]
-    if "question" in doc and "options" in doc:  # original operation
-        question = construct_prompt(doc, post_prompt)
-        if config["metadata"]["interleaved_format"]:
-            question = replace_images_tokens(question)
-    return question
-
-
-def mmmu_pro_doc_to_visual(doc):
-    if "question" in doc and "options" in doc:  # original operation
-        prompt = construct_prompt(doc)
-        image_tokens = re.findall(r"<image \d+>", prompt)
-        # Remove <> and  swap space as _
-        image_tokens = sorted(list(set([image_token.strip("<>").replace(" ", "_") for image_token in image_tokens])))
-        visual = [doc[image_token].convert("RGB") for image_token in image_tokens]
-    else:  # vision-only operation
-        visual = [doc["image"].convert("RGB")]
-    return visual
-
-
-# MMMU-PRO's all questions are multiple-choice questions
-def mmmu_pro_process_results(doc, results):
-    pred = results[0]
-    if "question" in doc and "options" in doc:
-        index2ans, all_choices = get_multi_choice_info(ast.literal_eval(doc["options"]))
-        parsed_pred = parse_multi_choice_response(pred, all_choices, index2ans)
-    else:
-        # Find all "Answer: X" patterns and take the last one
-        index2ans, all_choices = get_multi_choice_info(ast.literal_eval(doc["options"]))
-        answer_matches = re.findall(r"Answer:\s*([A-Z])\b(?![A-Z])", pred)
-        parsed_pred = answer_matches[-1] if answer_matches else ""
-        #print(f"Pred: {pred[-100:]}")
-        #print(f"Parsed prediction: {parsed_pred} and answer: {doc['answer']}")
-
-    mmmu_acc = {"id": doc["id"], "subject": doc["subject"], "answer": doc["answer"], "parsed_pred": parsed_pred}
-    return {"mmmu_acc": mmmu_acc}
-
-
-def mmmu_pro_composite_process_results(doc, results):
-    pred = results[0]
-    gt_list = ast.literal_eval(doc["answers"])
-    # Parse out option letters from the prediction
-    option_letters = re.findall(r"\b[A-Z]\b", pred)
-
-    cutout_letters = option_letters[: len(gt_list)]
-
-    # If cutout_letters is shorter than gt_list, pad with empty strings
-    while len(cutout_letters) < len(gt_list):
-        cutout_letters.append("")
-
-    assert len(cutout_letters) == len(gt_list), f"Mismatch in lengths: cutout_letters ({len(cutout_letters)}) != gt_list ({len(gt_list)})"
-
-    mmmu_acc = {"id": doc["id"], "subject": doc["subject"], "answer": gt_list, "parsed_pred": cutout_letters}
-    return {"mmmu_acc": mmmu_acc}
-
-
-def mmmu_pro_aggregate_results(results):
-    evaluation_result = {}
-    subset_to_eval_samples = defaultdict(list)
-    for result in results:
-        subset_to_eval_samples[result["subject"]].append(result)
-
-    for subset, sub_eval_samples in subset_to_eval_samples.items():
-        judge_dict, metric_dict = evaluate_mmmu(sub_eval_samples)
-        metric_dict.update({"num_example": len(sub_eval_samples)})
-        evaluation_result[subset] = metric_dict
-
-    printable_results = {}
-    for domain, in_domain_cats in DOMAIN_CAT2SUB_CAT.items():
-        in_domain_cat_results = {}
-        for cat_name in in_domain_cats:
-            if cat_name in evaluation_result.keys():
-                in_domain_cat_results[cat_name] = evaluation_result[cat_name]
-            else:
-                pass
-        in_domain_ins_acc = calculate_ins_level_acc(in_domain_cat_results)
-        in_domain_data_num = sum([cat_results["num_example"] for cat_results in in_domain_cat_results.values()])
-        printable_results["Overall-" + domain] = {
-            "num": int(in_domain_data_num),
-            "acc": round(in_domain_ins_acc, 5),
-        }
-        # add sub category
-        for cat_name, cat_results in in_domain_cat_results.items():
-            printable_results[cat_name] = {
-                "num": int(cat_results["num_example"]),
-                "acc": round(cat_results["acc"], 5),
-            }
-
-    all_ins_acc = calculate_ins_level_acc(evaluation_result)
-    printable_results["Overall"] = {
-        "num": sum([cat_results["num_example"] for cat_results in evaluation_result.values()]),
-        "acc": round(all_ins_acc, 5),
-    }
-    print(printable_results)
-    return printable_results["Overall"]["acc"]
-
-
-##################
-# Helper functions written by official MMMU repo.
-##################
-
-
-def calculate_ins_level_acc(results):
-    """Calculate the instruction level accuracy for given Subject results
-    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L246
-    """
-    acc = 0
-    ins_num = 0
-    for cat_results in results.values():
-        acc += cat_results["acc"] * cat_results["num_example"]
-        ins_num += cat_results["num_example"]
-    if ins_num == 0:
-        return 0
-    return acc / ins_num
-
-
-DOMAIN_CAT2SUB_CAT = {
-    "Art and Design": ["Art", "Art_Theory", "Design", "Music"],
-    "Business": ["Accounting", "Economics", "Finance", "Manage", "Marketing"],
-    "Science": [
-        "Biology",
-        "Chemistry",
-        "Geography",
-        "Math",
-        "Physics",
-    ],
-    "Health and Medicine": [
-        "Basic_Medical_Science",
-        "Clinical_Medicine",
-        "Diagnostics_and_Laboratory_Medicine",
-        "Pharmacy",
-        "Public_Health",
-    ],
-    "Humanities and Social Science": [
-        "History",
-        "Literature",
-        "Sociology",
-        "Psychology",
-    ],
-    "Tech and Engineering": [
-        "Agriculture",
-        "Architecture_and_Engineering",
-        "Computer_Science",
-        "Electronics",
-        "Energy_and_Power",
-        "Materials",
-        "Mechanical_Engineering",
-    ],
-}
-
-
-def eval_multi_choice(gold_i, pred_i):
+# Helper function to load image from bytes if necessary, or return PIL Image directly
+def _load_image(image_data):
     """
-    Evaluate a multiple choice instance.
-    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L175
+    Loads an image from various formats provided by the Hugging Face datasets library.
+    Assumes image_data can be a dictionary with 'bytes' key or a PIL Image object.
     """
-    correct = False
-    # only they are exactly the same, we consider it as correct
-    if isinstance(gold_i, list):
-        for answer in gold_i:
-            if answer == pred_i:
-                correct = True
-                break
-    else:  # gold_i is a string
-        if gold_i == pred_i:
-            correct = True
-    return correct
+    if isinstance(image_data, dict) and 'bytes' in image_data:
+        return Image.open(io.BytesIO(image_data['bytes']))
+    elif isinstance(image_data, Image.Image):
+        return image_data
+    return None # Return None for unsupported formats or missing data
 
-
-def eval_open(gold_i, pred_i):
+def blink_doc_to_text_with_disclaimer(doc):
     """
-    Evaluate an open question instance
-    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L191
+    Constructs the textual input for the LMM from a BLINK document.
+    Combines the 'question' and 'prompt' fields and lists the multiple-choice options.
     """
-    correct = False
-    if isinstance(gold_i, list):
-        # use float to avoid trivial matches
-        norm_answers = []
-        for answer in gold_i:
-            norm_answers.extend(normalize_str(answer))
-    else:
-        norm_answers = normalize_str(gold_i)
-    for pred in pred_i:  # pred is already normalized in parse response phase
-        if isinstance(pred, str):  # if it's a string, then find if ans in the pred_i
-            for norm_ans in norm_answers:
-                # only see if the string answer in the string pred
-                if isinstance(norm_ans, str) and norm_ans in pred:
-                    if not correct:
-                        correct = True
-                    break
-        else:  # it's a float number
-            if pred in norm_answers:
-                if not correct:
-                    correct = True
-                break
-    return correct
+    return blink_doc_to_text(doc, need_disclaimer=True)
 
 
-def evaluate_mmmu(samples):
+def blink_doc_to_text(doc, need_disclaimer: bool = False):
     """
-    Batch evaluation for multiple choice and open questions.
-    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L219
+    Constructs the textual input for the LMM from a BLINK document.
+    Combines the 'question' and 'prompt' fields and lists the multiple-choice options.
     """
-    total_count = 0
-    pred_correct = 0
-    judge_dict = dict()
-    for sample in samples:
-        if isinstance(sample["answer"], str) and isinstance(sample["parsed_pred"], str):
-            gold_i = sample["answer"]
-            pred_i = sample["parsed_pred"]
-            correct = eval_multi_choice(gold_i, pred_i)
-            total_count += 1
-            if correct:
-                judge_dict[sample["id"]] = "Correct"
-                pred_correct += 1
-            else:
-                judge_dict[sample["id"]] = "Wrong"
-        elif isinstance(sample["answer"], list) and isinstance(sample["parsed_pred"], list):
-            for gold, pred in zip(sample["answer"], sample["parsed_pred"]):
-                correct = eval_multi_choice(gold, pred)
-                total_count += 1
-                if correct:
-                    judge_dict[sample["id"]] = "Correct"
-                    pred_correct += 1
-                else:
-                    judge_dict[sample["id"]] = "Wrong"
-
-    if total_count == 0:
-        return {"acc": 0}
-    return judge_dict, {"acc": pred_correct / total_count}
+    prompt = doc['prompt']
+    if need_disclaimer:
+        prompt = disclaimer + prompt
+    prompt += ". The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of options. Think step by step before answering."
+    return prompt 
 
-
-def parse_multi_choice_response(response, all_choices, index2ans):
+def blink_doc_to_target(doc):
     """
-    Parse the prediction from the generated response.
-    Return the predicted index e.g., A, B, C, D.
-    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L10
+    Extracts the ground truth answer from a BLINK document.
     """
-    for char in [",", ".", "!", "?", ";", ":", "'"]:
-        response = response.strip(char)
-    response = " " + response + " "  # add space to avoid partial match
-
-    index_ans = True
-    ans_with_brack = False
-    candidates = []
-    for choice in all_choices:  # e.g., (A) (B) (C) (D)
-        if f"({choice})" in response:
-            candidates.append(choice)
-            ans_with_brack = True
-
-    if len(candidates) == 0:
-        for choice in all_choices:  # e.g., A B C D
-            if f"{choice} " in response:
-                candidates.append(choice)
-
-    if len(candidates) == 0:
-        for choice in all_choices:  # e.g., A. B. C. D.
-            if f"{choice}." in response:
-                candidates.append(choice)
+    # The target is the correct answer from the choices.
+    return doc['answer']
 
-    # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example
-    if len(candidates) == 0 and len(response.split()) > 5:
-        for index, ans in index2ans.items():
-            if ans.lower() in response.lower():
-                candidates.append(index)
-                index_ans = False  # it's content ans.
-
-    if len(candidates) == 0:  # still not get answer, randomly choose one.
-        pred_index = random.choice(all_choices)
-    elif len(candidates) > 1:
-        start_indexes = []
-        if index_ans:
-            if ans_with_brack:
-                for can in candidates:
-                    index = response.rfind(f"({can})")
-                    start_indexes.append(index)  # -1 will be ignored anyway
-                # start_indexes = [generated_response.index(f'({can})') for can in candidates]
-            else:
-                for can in candidates:
-                    index = response.rfind(f" {can} ")
-                    start_indexes.append(index)
-        else:
-            for can in candidates:
-                index = response.lower().rfind(index2ans[can].lower())
-                start_indexes.append(index)
-        # get the last one
-        pred_index = candidates[np.argmax(start_indexes)]
-    else:  # if only one candidate, use it.
-        pred_index = candidates[0]
-
-    return pred_index
-
-
-def extract_numbers(string):
+def blink_doc_to_visual(doc):
     """
-    Exact all forms of numbers from a string with regex.
-    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L100
+    Loads and prepares images associated with a BLINK document.
+    Handles multiple image columns (image_1, image_2, image_3, image_4).
     """
-    # Pattern for numbers with commas
-    pattern_commas = r"-?\b\d{1,3}(?:,\d{3})+\b"
-    # Pattern for scientific notation
-    pattern_scientific = r"-?\d+(?:\.\d+)?[eE][+-]?\d+"
-    # Pattern for simple numbers without commas
-    pattern_simple = r"-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])"
-
-    # Extract numbers with commas
-    numbers_with_commas = re.findall(pattern_commas, string)
-    # Extract numbers in scientific notation
-    numbers_scientific = re.findall(pattern_scientific, string)
-    # Extract simple numbers without commas
-    numbers_simple = re.findall(pattern_simple, string)
+    image_paths = []
+    for k in ['image_1', 'image_2', 'image_3', 'image_4']:
+        if k in doc and doc[k]:
+            img = doc[k]
+            if img:
+                image_paths.append(img)
 
-    # Combine all extracted numbersz
-    all_numbers = numbers_with_commas + numbers_scientific + numbers_simple
-    return all_numbers
+    return image_paths if image_paths else None # Return list of PIL Images or None
 
-
-def check_is_number(string):
+def blink_process_results(doc, results):
     """
-    Check if the given string a number.
-    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L65
+    Processes the model's generated results and calculates instance-level accuracy.
     """
+    # 'results' will be a list of strings generated by the model.
+    # For multiple-choice, we expect the model to generate one of the choices.
+    all_choices = ['(A)', '(B)', '(C)', '(D)', '(E)'][:len(doc['choices'])]
     try:
-        float(string.replace(",", ""))
-        return True
-    except ValueError:
-        # check if there's comma inside
-        return False
-
-
-def normalize_str(string):
-    """
-    Normalize the str to lower case and make them float numbers if possible.
-    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L76
-    """
-    # check if characters in the string
-
-    # if number, numerize it.
-    string = string.strip()
+        results = results[0]
+        answer_matches = re.findall(r"Answer:\s*([A-Z])\b(?![A-Z])", results)
+        parsed_pred = answer_matches[-1] if answer_matches else ""
 
-    is_number = check_is_number(string)
+        if parsed_pred in ["A", "B", "C", "D", "E"]:
+            prediction = "(" + parsed_pred + ")"
+        elif parsed_pred in ['(A)', '(B)', '(C)', '(D)', '(E)']:
+            prediction = parsed_pred
+        else:
+            print("Error parsing prediction")
+            prediction = "N/A"
+    except Exception as e:
+        print("Error processing results from blink", e)
+        prediction = "N/A"
+        #import pdb; pdb.set_trace()
+    #print("doc with answer", doc['answer'], "and prediction", prediction, "doc", doc)
+    blink_acc = {"id": doc["idx"], "sub_task": doc["sub_task"], "answer": doc["answer"], "parsed_pred": prediction, "results": results}
+    return {"blink_acc": blink_acc}
 
-    if is_number:
-        string = string.replace(",", "")
-        string = float(string)
-        # leave 2 decimal
-        string = round(string, 2)
-        return [string]
-    else:  # it's likely to be a string
-        # lower it
-        string = string.lower()
-        if len(string) == 1:
-            return [" " + string, string + " "]  # avoid trivial matches
-        return [string]
+    
 
 
-def parse_open_response(response):
+def blink_aggregation(results):
     """
-    Parse the prediction from the generated response.
-    Return a list of predicted strings or numbers.
-    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L122
-    """
-
-    # content = content.strip("\n").strip(".").strip(" ")
-    def get_key_subresponses(response):
-        key_responses = []
-        response = response.strip().strip(".").lower()
-        sub_responses = re.split(r"\.\s(?=[A-Z])|\n", response)
-        indicators_of_keys = [
-            "could be ",
-            "so ",
-            "is ",
-            "thus ",
-            "therefore ",
-            "final ",
-            "answer ",
-            "result ",
-        ]
-        key_responses = []
-        for index, resp in enumerate(sub_responses):
-            # if last one, accept it's an equation (the entire response can be just one sentence with equation)
-            if index == len(sub_responses) - 1:
-                indicators_of_keys.extend(["="])
-            shortest_key_response = None  # the shortest response that may contain the answer (tail part of the response)
-            for indicator in indicators_of_keys:
-                if indicator in resp:
-                    if not shortest_key_response:
-                        shortest_key_response = resp.split(indicator)[-1].strip()
-                    else:
-                        if len(resp.split(indicator)[-1].strip()) < len(shortest_key_response):
-                            shortest_key_response = resp.split(indicator)[-1].strip()
-                    # key_responses.append(resp.split(indicator)[1].strip())
+    Computes accuracy and number of examples from a list of result dictionaries.
+    
+    Each dictionary should have at least:
+    - 'answer': the ground truth
+    - 'parsed_pred': the model prediction
 
-            if shortest_key_response:
-                # and it's not trivial
-                if shortest_key_response.strip() not in [
-                    ":",
-                    ",",
-                    ".",
-                    "!",
-                    "?",
-                    ";",
-                    ":",
-                    "'",
-                ]:
-                    key_responses.append(shortest_key_response)
-        if len(key_responses) == 0:  # did not found any
-            return [response]
-        return key_responses
+    Args:
+        results (list of dict): List of prediction results.
 
-    # pdb.set_trace()
-    key_responses = get_key_subresponses(response)
-
-    pred_list = key_responses.copy()  # keep the original string response
-    for resp in key_responses:
-        pred_list.extend(extract_numbers(resp))
-
-    tmp_pred_list = []
-    for i in range(len(pred_list)):
-        tmp_pred_list.extend(normalize_str(pred_list[i]))
-    pred_list = tmp_pred_list
-
-    # remove duplicates
-    pred_list = list(set(pred_list))
-
-    return pred_list
-
-
-def get_multi_choice_info(options):
-    """
-    Given the list of options for multiple choice question
-    Return the index2ans and all_choices
-    https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/data_utils.py#L54
+    Returns:
+        dict: A dictionary with 'accuracy' and 'num_examples'.
     """
+    if not results:
+        return {'accuracy': 0.0, 'num_examples': 0}
+
+    correct = 0
+    for entry in results:
+        if entry['parsed_pred'] == entry['answer']:
+            correct += 1
 
-    start_chr = "A"
-    all_choices = []
-    index2ans = {}
-    for i, option in enumerate(options):
-        index2ans[chr(ord(start_chr) + i)] = option
-        all_choices.append(chr(ord(start_chr) + i))
+    num_examples = len(results)
+    accuracy = correct / num_examples
 
-    return index2ans, all_choices
+    print("Number of examples:", num_examples, "Accuracy:", accuracy)
+    return accuracy
\ No newline at end of file
diff --git a/lmms_eval/tasks/mmmu_pro/mmmu_pro_cot.yaml b/lmms_eval/tasks/mmmu_pro/mmmu_pro_cot.yaml
index 4c4b1950c..b2b8cb281 100755
--- a/lmms_eval/tasks/mmmu_pro/mmmu_pro_cot.yaml
+++ b/lmms_eval/tasks/mmmu_pro/mmmu_pro_cot.yaml
@@ -1,6 +1,6 @@
 group: mmmu_pro_cot
 task:
 - mmmu_pro_standard_cot
-- mmmu_pro_vision_cot
+#- mmmu_pro_vision_cot
 # - mmmu_pro_composite_cot
 #- mmmu_pro_original_cot
diff --git a/lmms_eval/tasks/mmmu_pro/mmmu_pro_standard_cot.yaml b/lmms_eval/tasks/mmmu_pro/mmmu_pro_standard_cot.yaml
index 0c889a3b8..9ff170926 100755
--- a/lmms_eval/tasks/mmmu_pro/mmmu_pro_standard_cot.yaml
+++ b/lmms_eval/tasks/mmmu_pro/mmmu_pro_standard_cot.yaml
@@ -3,8 +3,9 @@ dataset_path: MMMU/MMMU_Pro
 dataset_name: "standard (10 options)"
 test_split: test
 output_type: generate_until
+#process_docs: !function utils.process_docs
 doc_to_visual: !function utils.mmmu_pro_doc_to_visual
-doc_to_text: "Write out the multiple-choice question in the image and then solve it. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of options. Think step by step before answering."
+doc_to_text: "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of options. Think step by step before answering."
 doc_to_target: "{{answer}}"
 # The return value of process_results will be used by metrics
 process_results: !function utils.mmmu_pro_process_results
diff --git a/lmms_eval/tasks/mmmu_pro/utils.py b/lmms_eval/tasks/mmmu_pro/utils.py
index e58c9aa46..46c71777c 100755
--- a/lmms_eval/tasks/mmmu_pro/utils.py
+++ b/lmms_eval/tasks/mmmu_pro/utils.py
@@ -29,7 +29,7 @@ def process_docs(docs):
     """
     # logger.info(f"processing docs")
     # Process images in place
-    docs = docs.select(range(65)) # filter out some samples!
+    docs = docs.select(range(1)) # filter out some samples!
     return docs
 
 def replace_images_tokens(input_string):