diff --git a/calibration/step-1-prepare-calibration-dataset.py b/calibration/step-1-prepare-calibration-dataset.py
index 374c086a..29d95447 100755
--- a/calibration/step-1-prepare-calibration-dataset.py
+++ b/calibration/step-1-prepare-calibration-dataset.py
@@ -38,6 +38,9 @@ def main(args):
             padding_side="left",
             use_fast=False,
         )
+        # Llama4 models in Text Only mode fail with False if use_fast=False
+        if isinstance(tokenizer, bool):
+            raise ValueError("Tokenizer is a boolean")
     except (OSError, ValueError, RuntimeError, ImportError):
         tokenizer = transformers.AutoTokenizer.from_pretrained(
             args.model,