Make analyzer REST languages configurable with SUPPORTED_LANGUAGES

CHANGELOG.md

@@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file.
 
 ### Analyzer
 #### Fixed
+- Made REST analyzer supported languages configurable through the `SUPPORTED_LANGUAGES` environment variable, allowing `/supportedentities` and `/recognizers` to expose non-English recognizers.
 - Language model recognizers (`BasicLangExtractRecognizer`, `AzureOpenAILangExtractRecognizer`) configured in a recognizer registry YAML now honour `config_path` (and other recognizer-specific kwargs). Previously these entries were validated by the strict `PredefinedRecognizerConfig` schema, which has no `config_path` field and does not allow extra keys, so `config_path` was silently dropped and the recognizer fell back to its bundled default model configuration. Added a `LangExtractRecognizerConfig` model (`extra="allow"`) and registered both recognizer class names in `CONFIG_MODEL_MAP`.
 - `BasicLangExtractRecognizer` now honours values under `langextract.model.provider.language_model_params` (including `timeout` and `num_ctx`). Previously these were silently dropped because `langextract.extract()` ignores its `language_model_params` argument when a pre-built `ModelConfig` is passed via `config=`, causing Ollama-backed recognizers to fall back to langextract's 120s default regardless of the configured timeout. The recognizer now merges `language_model_params` into `ModelConfig.provider_kwargs`, which is the path that reaches the provider constructor. Explicit entries under `provider.kwargs:` still take precedence. Also fixed a `TypeError` when `kwargs:` or `language_model_params:` is `null` in the YAML. (#1943, Thanks @lsternlicht)
 

presidio-analyzer/app.py

@@ -22,6 +22,20 @@
 
 LOGGING_CONF_FILE = "logging.ini"
 
+
+def _get_supported_languages():
+    supported_languages = os.environ.get("SUPPORTED_LANGUAGES")
+    if not supported_languages:
+        return None
+
+    parsed_languages = [
+        language.strip()
+        for language in supported_languages.split(",")
+        if language.strip()
+    ]
+    return parsed_languages or None
+
+
 WELCOME_MESSAGE = r"""
  _______  _______  _______  _______ _________ ______  _________ _______
 (  ____ )(  ____ )(  ____ \(  ____ \\__   __/(  __  \ \__   __/(  ___  )
@@ -48,12 +62,14 @@ def __init__(self):
         recognizer_registry_conf_file = (
             os.environ.get("RECOGNIZER_REGISTRY_CONF_FILE") or None
         )
+        supported_languages = _get_supported_languages()
 
         self.logger.info("Starting analyzer engine")
         self.engine: AnalyzerEngine = AnalyzerEngineProvider(
             analyzer_engine_conf_file=analyzer_conf_file,
             nlp_engine_conf_file=nlp_engine_conf_file,
             recognizer_registry_conf_file=recognizer_registry_conf_file,
+            supported_languages=supported_languages,
         ).create_engine()
 
         self.batch_engine = BatchAnalyzerEngine(self.engine)
@@ -86,7 +102,7 @@ def analyze() -> Tuple[str, int]:
                     texts=batch,
                     batch_size=min(
                         len(batch),
-                        int(os.environ.get("BATCH_SIZE", DEFAULT_BATCH_SIZE))
+                        int(os.environ.get("BATCH_SIZE", DEFAULT_BATCH_SIZE)),
                     ),
                     language=req_data.language,
                     correlation_id=req_data.correlation_id,
@@ -99,9 +115,8 @@ def analyze() -> Tuple[str, int]:
                     allow_list_match=req_data.allow_list_match,
                     regex_flags=req_data.regex_flags,
                     n_process=min(
-                        len(batch),
-                        int(os.environ.get("N_PROCESS", DEFAULT_N_PROCESS))
-                    )
+                        len(batch), int(os.environ.get("N_PROCESS", DEFAULT_N_PROCESS))
+                    ),
                 )
                 results = []
                 for recognizer_result_list in iterator:

presidio-analyzer/presidio_analyzer/analyzer_engine_provider.py

@@ -22,13 +22,15 @@ class AnalyzerEngineProvider:
     :param nlp_engine_conf_file: the path to the nlp engine configuration file
     :param recognizer_registry_conf_file: the path to the recognizer
     registry configuration file
+    :param supported_languages: optional runtime override for supported languages
     """
 
     def __init__(
         self,
         analyzer_engine_conf_file: Optional[Union[Path, str]] = None,
         nlp_engine_conf_file: Optional[Union[Path, str]] = None,
         recognizer_registry_conf_file: Optional[Union[Path, str]] = None,
+        supported_languages: Optional[List[str]] = None,
     ):
         if analyzer_engine_conf_file:
             ConfigurationValidator.validate_file_path(analyzer_engine_conf_file)
@@ -40,6 +42,7 @@ def __init__(
         self.configuration = self.get_configuration(conf_file=analyzer_engine_conf_file)
         self.nlp_engine_conf_file = nlp_engine_conf_file
         self.recognizer_registry_conf_file = recognizer_registry_conf_file
+        self.supported_languages = supported_languages
 
     def get_configuration(
         self, conf_file: Optional[Union[Path, str]]
@@ -61,8 +64,7 @@ def get_configuration(
                     configuration = yaml.safe_load(file)
             except OSError:
                 logger.warning(
-                    f"configuration file {conf_file} not found.  "
-                    f"Using default config."
+                    f"configuration file {conf_file} not found.  Using default config."
                 )
                 with open(self._get_full_conf_path()) as file:
                     configuration = yaml.safe_load(file)
@@ -86,7 +88,11 @@ def create_engine(self) -> AnalyzerEngine:
         """
 
         nlp_engine = self._load_nlp_engine()
-        supported_languages = self.configuration.get("supported_languages", ["en"])
+        supported_languages = (
+            self.supported_languages
+            if self.supported_languages is not None
+            else self.configuration.get("supported_languages", ["en"])
+        )
         default_score_threshold = self.configuration.get("default_score_threshold", 0)
 
         registry = self._load_recognizer_registry(

presidio-analyzer/tests/test_analyzer_engine_provider.py

@@ -38,8 +38,7 @@ def test_analyzer_engine_provider_default_configuration(mandatory_recognizers):
     )
     assert engine.default_score_threshold == 0
     assert all(
-        recognizer.score_thresholds == {}
-        for recognizer in engine.registry.recognizers
+        recognizer.score_thresholds == {} for recognizer in engine.registry.recognizers
     )
     names = [recognizer.name for recognizer in engine.registry.recognizers]
     for predefined_recognizer in mandatory_recognizers:
@@ -108,6 +107,18 @@ def test_analyzer_engine_provider_configuration_file():
     assert engine.nlp_engine.engine_name == "spacy"
 
 
+def test_analyzer_engine_provider_supported_languages_override():
+    test_yaml, _, _ = get_full_paths("conf/test_analyzer_engine.yaml")
+    provider = AnalyzerEngineProvider(
+        analyzer_engine_conf_file=test_yaml,
+        supported_languages=["it", "es"],
+    )
+
+    engine = provider.create_engine()
+
+    assert engine.supported_languages == ["it", "es"]
+
+
 def test_analyzer_engine_provider_inline_recognizer_thresholds_affect_output(tmp_path):
     analyzer_yaml, _, _ = get_full_paths("conf/test_analyzer_engine.yaml")
 
@@ -149,9 +160,7 @@ def test_analyzer_engine_provider_external_registry_thresholds_affect_output(tmp
                 "default_score_threshold": 0.9,
                 "nlp_configuration": {
                     "nlp_engine_name": "spacy",
-                    "models": [
-                        {"lang_code": "en", "model_name": "en_core_web_lg"}
-                    ],
+                    "models": [{"lang_code": "en", "model_name": "en_core_web_lg"}],
                 },
             }
         )
@@ -193,8 +202,7 @@ def test_analyzer_engine_provider_defaults(mandatory_recognizers):
     assert engine.supported_languages == ["en"]
     assert engine.default_score_threshold == 0
     assert all(
-        recognizer.score_thresholds == {}
-        for recognizer in engine.registry.recognizers
+        recognizer.score_thresholds == {} for recognizer in engine.registry.recognizers
     )
     recognizer_registry = engine.registry
     assert (
@@ -271,7 +279,10 @@ def analyze(
 
     assert len(analyzer_engine.analyze("This is a test", language="en")) > 0
 
-@pytest.mark.skipif(pytest.importorskip("azure"), reason="Optional dependency not installed") # noqa: E501
+
+@pytest.mark.skipif(
+    pytest.importorskip("azure"), reason="Optional dependency not installed"
+)  # noqa: E501
 def test_analyzer_engine_provider_with_ahds():
     analyzer_yaml, _, _ = get_full_paths(
         "conf/test_ahds_reco.yaml",
@@ -300,7 +311,6 @@ def analyze(
     assert len(ahds_recognizers) == 1
 
     assert len(analyzer_engine.analyze("This is a test", language="en")) > 0
-
 
 
 def test_analyzer_engine_provider_no_nlp_recognizer():
@@ -316,7 +326,14 @@ def test_analyzer_engine_provider_no_nlp_recognizer():
     recognizer = analyzer_engine.get_recognizers()[0]
     assert isinstance(recognizer, CreditCardRecognizer)
 
-    assert len(analyzer_engine.analyze("My Credit card number is 4917300800000000", language="en")) > 0
+    assert (
+        len(
+            analyzer_engine.analyze(
+                "My Credit card number is 4917300800000000", language="en"
+            )
+        )
+        > 0
+    )
 
 
 def test_analyzer_engine_provider_no_nlp_recognizer_is_added():
@@ -344,7 +361,9 @@ def test_analyzer_engine_provider_no_nlp_recognizer_is_added_per_language():
 
     analyzer_engine = provider.create_engine()
 
-    assert len(analyzer_engine.get_recognizers()) == 4 # Two CreditCardRecognizers and two SpacyRecognizers
+    assert (
+        len(analyzer_engine.get_recognizers()) == 4
+    )  # Two CreditCardRecognizers and two SpacyRecognizers
     nlp_recognizers = [
         rec
         for rec in analyzer_engine.get_recognizers()
@@ -372,7 +391,8 @@ def test_analyzer_engine_provider_multiple_nlp_recognizers_raises_exception():
     with pytest.raises(
         ValueError,
         match=f"Multiple NLP recognizers for language en found in the configuration. "
-                f"Please remove the duplicates."):
+        f"Please remove the duplicates.",
+    ):
         provider = AnalyzerEngineProvider(analyzer_engine_conf_file=analyzer_yaml)
         analyzer_engine = provider.create_engine()
 
@@ -385,7 +405,9 @@ def test_analyzer_engine_provider_no_nlp_engine_or_provider_results_in_default_n
 
     analyzer_engine = provider.create_engine()
 
-    assert len(analyzer_engine.get_recognizers()) == 2 # SpacyRecognizer, CreditCardRecognizer
+    assert (
+        len(analyzer_engine.get_recognizers()) == 2
+    )  # SpacyRecognizer, CreditCardRecognizer
     nlp_recognizer = [
         rec
         for rec in analyzer_engine.get_recognizers()
@@ -418,6 +440,7 @@ def test_analyzer_engine_stanza_without_recognizer_creates_recognizer():
     }
     assert supported_languages == {"en", "es"}
 
+
 def test_analyzer_engine_provider_one_custom_recognizer():
     analyzer_yaml, _, _ = get_full_paths(
         "conf/custom_recognizer_yaml.yaml",
@@ -426,7 +449,9 @@ def test_analyzer_engine_provider_one_custom_recognizer():
 
     analyzer_engine = provider.create_engine()
     assert len(analyzer_engine.get_recognizers()) == 1
-    assert analyzer_engine.analyze("My zip code is 12345", language="en")[0].score == pytest.approx(0.4)
+    assert analyzer_engine.analyze("My zip code is 12345", language="en")[
+        0
+    ].score == pytest.approx(0.4)
 
 
 def test_analyzer_engine_provider_invalid_analyzer_conf_file():
@@ -444,7 +469,9 @@ def test_analyzer_engine_provider_invalid_nlp_conf_file():
 def test_analyzer_engine_provider_invalid_registry_conf_file():
     """Test that invalid recognizer registry configuration file path raises error."""
     with pytest.raises(ValueError):
-        AnalyzerEngineProvider(recognizer_registry_conf_file="/nonexistent/path/file.yaml")
+        AnalyzerEngineProvider(
+            recognizer_registry_conf_file="/nonexistent/path/file.yaml"
+        )
 
 
 def test_analyzer_engine_provider_get_configuration_with_nonexistent_file():
@@ -465,7 +492,7 @@ def test_analyzer_engine_provider_get_configuration_with_invalid_yaml():
     import tempfile
 
     # Create a temporary file with invalid YAML
-    with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f:
         f.write("invalid: yaml: content: [[[")
         temp_file = f.name
 
@@ -703,8 +730,6 @@ def test_analyzer_engine_provider_configuration_logging(caplog):
     assert len(caplog.records) > 0
 
 
-
-
 # --- install_models / _install_models_from_nlp_config tests ---
 
 _NLP_CONF_CONTENT = (
@@ -744,7 +769,9 @@ def test_install_models_analyzer_conf_without_nlp_falls_back_to_nlp_conf(tmp_pat
     nlp_yaml.write_text(_NLP_CONF_CONTENT)
 
     with patch("install_nlp_models._download_model") as mock_dl:
-        install_models(nlp_conf_file=str(nlp_yaml), analyzer_conf_file=str(analyzer_yaml))
+        install_models(
+            nlp_conf_file=str(nlp_yaml), analyzer_conf_file=str(analyzer_yaml)
+        )
 
     mock_dl.assert_called_once_with("spacy", "en_core_web_sm")
 
@@ -768,7 +795,9 @@ def test_install_models_analyzer_conf_takes_priority_over_nlp_conf(tmp_path):
     nlp_yaml.write_text(_NLP_CONF_CONTENT)
 
     with patch("install_nlp_models._download_model") as mock_dl:
-        install_models(nlp_conf_file=str(nlp_yaml), analyzer_conf_file=str(analyzer_yaml))
+        install_models(
+            nlp_conf_file=str(nlp_yaml), analyzer_conf_file=str(analyzer_yaml)
+        )
 
     mock_dl.assert_called_once_with("spacy", "en_core_web_lg")
 

presidio-analyzer/tests/test_app.py

@@ -0,0 +1,51 @@
+# ruff: noqa: D103
+
+from types import SimpleNamespace
+from unittest.mock import Mock, patch
+
+from app import Server, _get_supported_languages
+
+
+def test_get_supported_languages_from_environment(monkeypatch):
+    monkeypatch.setenv("SUPPORTED_LANGUAGES", " en, es ,it ")
+
+    assert _get_supported_languages() == ["en", "es", "it"]
+
+
+def test_get_supported_languages_is_unset(monkeypatch):
+    monkeypatch.delenv("SUPPORTED_LANGUAGES", raising=False)
+
+    assert _get_supported_languages() is None
+
+
+def test_server_passes_supported_languages_to_provider(monkeypatch):
+    monkeypatch.setenv("SUPPORTED_LANGUAGES", "en,es,it")
+    provider = Mock()
+    provider.return_value.create_engine.return_value = SimpleNamespace()
+
+    with (
+        patch("app.AnalyzerEngineProvider", provider),
+        patch("app.BatchAnalyzerEngine"),
+    ):
+        Server()
+
+    assert provider.call_args.kwargs["supported_languages"] == ["en", "es", "it"]
+
+
+def test_supported_entities_uses_configured_language(monkeypatch):
+    monkeypatch.setenv("SUPPORTED_LANGUAGES", "en,es")
+    engine = Mock()
+    engine.get_supported_entities.return_value = ["ES_NIF"]
+    provider = Mock()
+    provider.return_value.create_engine.return_value = engine
+
+    with (
+        patch("app.AnalyzerEngineProvider", provider),
+        patch("app.BatchAnalyzerEngine"),
+    ):
+        client = Server().app.test_client()
+        response = client.get("/supportedentities?language=es")
+
+    assert response.status_code == 200
+    assert response.get_json() == ["ES_NIF"]
+    engine.get_supported_entities.assert_called_once_with("es")