feat: add dolphin

docling/datamodel/pipeline_options_vlm_model.py

@@ -27,6 +27,7 @@ class TransformersModelType(str, Enum):
     AUTOMODEL = "automodel"
     AUTOMODEL_VISION2SEQ = "automodel-vision2seq"
     AUTOMODEL_CAUSALLM = "automodel-causallm"
+    AUTOMODEL_IMAGETEXTTOTEXT = "automodel-imagetexttotext"
 
 
 class InlineVlmOptions(BaseVlmOptions):

docling/models/vlm_models_inline/hf_transformers_model.py

@@ -41,6 +41,7 @@
             from transformers import (
                 AutoModel,
                 AutoModelForCausalLM,
+                AutoModelForImageTextToText,
                 AutoModelForVision2Seq,
                 AutoProcessor,
                 BitsAndBytesConfig,
@@ -91,6 +92,11 @@
                 == TransformersModelType.AUTOMODEL_VISION2SEQ
             ):
                 model_cls = AutoModelForVision2Seq
+            elif (
+                self.vlm_options.transformers_model_type
+                == TransformersModelType.AUTOMODEL_IMAGETEXTTOTEXT
+            ):
+                model_cls = AutoModelForImageTextToText
 
             self.processor = AutoProcessor.from_pretrained(
                 artifacts_path,
@@ -175,6 +181,12 @@
             _log.debug(f"prompt for {self.vlm_options.repo_id}: {prompt}")
 
             return prompt
+        if self.vlm_options.repo_id.lower().startswith("bytedance/dolphin"):
+            _log.debug("Using specialized prompt for dolphin")
+            # more info here https://huggingface.co/ByteDance/Dolphin
+            prompt = f"<s>{self.vlm_options.prompt} <Answer/>"
+            _log.debug(f"prompt for {self.vlm_options.repo_id}: {prompt}")
+            return prompt
 
         messages = [
             {