From 7edd30f8442d307db4ec8ea7b68ab773d269e113 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Sun, 19 Jul 2026 07:43:26 -0400 Subject: [PATCH] Expose configured analyzer languages through REST Signed-off-by: Rod Boev --- CHANGELOG.md | 1 + presidio-analyzer/app.py | 23 +++++-- .../analyzer_engine_provider.py | 12 +++- .../tests/test_analyzer_engine_provider.py | 69 +++++++++++++------ presidio-analyzer/tests/test_app.py | 51 ++++++++++++++ 5 files changed, 129 insertions(+), 27 deletions(-) create mode 100644 presidio-analyzer/tests/test_app.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 40890a66eb..b84d72b324 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file. ### Analyzer #### Fixed +- Made REST analyzer supported languages configurable through the `SUPPORTED_LANGUAGES` environment variable, allowing `/supportedentities` and `/recognizers` to expose non-English recognizers. - Language model recognizers (`BasicLangExtractRecognizer`, `AzureOpenAILangExtractRecognizer`) configured in a recognizer registry YAML now honour `config_path` (and other recognizer-specific kwargs). Previously these entries were validated by the strict `PredefinedRecognizerConfig` schema, which has no `config_path` field and does not allow extra keys, so `config_path` was silently dropped and the recognizer fell back to its bundled default model configuration. Added a `LangExtractRecognizerConfig` model (`extra="allow"`) and registered both recognizer class names in `CONFIG_MODEL_MAP`. - `BasicLangExtractRecognizer` now honours values under `langextract.model.provider.language_model_params` (including `timeout` and `num_ctx`). Previously these were silently dropped because `langextract.extract()` ignores its `language_model_params` argument when a pre-built `ModelConfig` is passed via `config=`, causing Ollama-backed recognizers to fall back to langextract's 120s default regardless of the configured timeout. The recognizer now merges `language_model_params` into `ModelConfig.provider_kwargs`, which is the path that reaches the provider constructor. Explicit entries under `provider.kwargs:` still take precedence. Also fixed a `TypeError` when `kwargs:` or `language_model_params:` is `null` in the YAML. (#1943, Thanks @lsternlicht) diff --git a/presidio-analyzer/app.py b/presidio-analyzer/app.py index 567a796cd0..b700284fdb 100644 --- a/presidio-analyzer/app.py +++ b/presidio-analyzer/app.py @@ -22,6 +22,20 @@ LOGGING_CONF_FILE = "logging.ini" + +def _get_supported_languages(): + supported_languages = os.environ.get("SUPPORTED_LANGUAGES") + if not supported_languages: + return None + + parsed_languages = [ + language.strip() + for language in supported_languages.split(",") + if language.strip() + ] + return parsed_languages or None + + WELCOME_MESSAGE = r""" _______ _______ _______ _______ _________ ______ _________ _______ ( ____ )( ____ )( ____ \( ____ \\__ __/( __ \ \__ __/( ___ ) @@ -48,12 +62,14 @@ def __init__(self): recognizer_registry_conf_file = ( os.environ.get("RECOGNIZER_REGISTRY_CONF_FILE") or None ) + supported_languages = _get_supported_languages() self.logger.info("Starting analyzer engine") self.engine: AnalyzerEngine = AnalyzerEngineProvider( analyzer_engine_conf_file=analyzer_conf_file, nlp_engine_conf_file=nlp_engine_conf_file, recognizer_registry_conf_file=recognizer_registry_conf_file, + supported_languages=supported_languages, ).create_engine() self.batch_engine = BatchAnalyzerEngine(self.engine) @@ -86,7 +102,7 @@ def analyze() -> Tuple[str, int]: texts=batch, batch_size=min( len(batch), - int(os.environ.get("BATCH_SIZE", DEFAULT_BATCH_SIZE)) + int(os.environ.get("BATCH_SIZE", DEFAULT_BATCH_SIZE)), ), language=req_data.language, correlation_id=req_data.correlation_id, @@ -99,9 +115,8 @@ def analyze() -> Tuple[str, int]: allow_list_match=req_data.allow_list_match, regex_flags=req_data.regex_flags, n_process=min( - len(batch), - int(os.environ.get("N_PROCESS", DEFAULT_N_PROCESS)) - ) + len(batch), int(os.environ.get("N_PROCESS", DEFAULT_N_PROCESS)) + ), ) results = [] for recognizer_result_list in iterator: diff --git a/presidio-analyzer/presidio_analyzer/analyzer_engine_provider.py b/presidio-analyzer/presidio_analyzer/analyzer_engine_provider.py index d03430a1f9..6b22584f1d 100644 --- a/presidio-analyzer/presidio_analyzer/analyzer_engine_provider.py +++ b/presidio-analyzer/presidio_analyzer/analyzer_engine_provider.py @@ -22,6 +22,7 @@ class AnalyzerEngineProvider: :param nlp_engine_conf_file: the path to the nlp engine configuration file :param recognizer_registry_conf_file: the path to the recognizer registry configuration file + :param supported_languages: optional runtime override for supported languages """ def __init__( @@ -29,6 +30,7 @@ def __init__( analyzer_engine_conf_file: Optional[Union[Path, str]] = None, nlp_engine_conf_file: Optional[Union[Path, str]] = None, recognizer_registry_conf_file: Optional[Union[Path, str]] = None, + supported_languages: Optional[List[str]] = None, ): if analyzer_engine_conf_file: ConfigurationValidator.validate_file_path(analyzer_engine_conf_file) @@ -40,6 +42,7 @@ def __init__( self.configuration = self.get_configuration(conf_file=analyzer_engine_conf_file) self.nlp_engine_conf_file = nlp_engine_conf_file self.recognizer_registry_conf_file = recognizer_registry_conf_file + self.supported_languages = supported_languages def get_configuration( self, conf_file: Optional[Union[Path, str]] @@ -61,8 +64,7 @@ def get_configuration( configuration = yaml.safe_load(file) except OSError: logger.warning( - f"configuration file {conf_file} not found. " - f"Using default config." + f"configuration file {conf_file} not found. Using default config." ) with open(self._get_full_conf_path()) as file: configuration = yaml.safe_load(file) @@ -86,7 +88,11 @@ def create_engine(self) -> AnalyzerEngine: """ nlp_engine = self._load_nlp_engine() - supported_languages = self.configuration.get("supported_languages", ["en"]) + supported_languages = ( + self.supported_languages + if self.supported_languages is not None + else self.configuration.get("supported_languages", ["en"]) + ) default_score_threshold = self.configuration.get("default_score_threshold", 0) registry = self._load_recognizer_registry( diff --git a/presidio-analyzer/tests/test_analyzer_engine_provider.py b/presidio-analyzer/tests/test_analyzer_engine_provider.py index 59df104e11..d3e8c86da1 100644 --- a/presidio-analyzer/tests/test_analyzer_engine_provider.py +++ b/presidio-analyzer/tests/test_analyzer_engine_provider.py @@ -38,8 +38,7 @@ def test_analyzer_engine_provider_default_configuration(mandatory_recognizers): ) assert engine.default_score_threshold == 0 assert all( - recognizer.score_thresholds == {} - for recognizer in engine.registry.recognizers + recognizer.score_thresholds == {} for recognizer in engine.registry.recognizers ) names = [recognizer.name for recognizer in engine.registry.recognizers] for predefined_recognizer in mandatory_recognizers: @@ -108,6 +107,18 @@ def test_analyzer_engine_provider_configuration_file(): assert engine.nlp_engine.engine_name == "spacy" +def test_analyzer_engine_provider_supported_languages_override(): + test_yaml, _, _ = get_full_paths("conf/test_analyzer_engine.yaml") + provider = AnalyzerEngineProvider( + analyzer_engine_conf_file=test_yaml, + supported_languages=["it", "es"], + ) + + engine = provider.create_engine() + + assert engine.supported_languages == ["it", "es"] + + def test_analyzer_engine_provider_inline_recognizer_thresholds_affect_output(tmp_path): analyzer_yaml, _, _ = get_full_paths("conf/test_analyzer_engine.yaml") @@ -149,9 +160,7 @@ def test_analyzer_engine_provider_external_registry_thresholds_affect_output(tmp "default_score_threshold": 0.9, "nlp_configuration": { "nlp_engine_name": "spacy", - "models": [ - {"lang_code": "en", "model_name": "en_core_web_lg"} - ], + "models": [{"lang_code": "en", "model_name": "en_core_web_lg"}], }, } ) @@ -193,8 +202,7 @@ def test_analyzer_engine_provider_defaults(mandatory_recognizers): assert engine.supported_languages == ["en"] assert engine.default_score_threshold == 0 assert all( - recognizer.score_thresholds == {} - for recognizer in engine.registry.recognizers + recognizer.score_thresholds == {} for recognizer in engine.registry.recognizers ) recognizer_registry = engine.registry assert ( @@ -271,7 +279,10 @@ def analyze( assert len(analyzer_engine.analyze("This is a test", language="en")) > 0 -@pytest.mark.skipif(pytest.importorskip("azure"), reason="Optional dependency not installed") # noqa: E501 + +@pytest.mark.skipif( + pytest.importorskip("azure"), reason="Optional dependency not installed" +) # noqa: E501 def test_analyzer_engine_provider_with_ahds(): analyzer_yaml, _, _ = get_full_paths( "conf/test_ahds_reco.yaml", @@ -300,7 +311,6 @@ def analyze( assert len(ahds_recognizers) == 1 assert len(analyzer_engine.analyze("This is a test", language="en")) > 0 - def test_analyzer_engine_provider_no_nlp_recognizer(): @@ -316,7 +326,14 @@ def test_analyzer_engine_provider_no_nlp_recognizer(): recognizer = analyzer_engine.get_recognizers()[0] assert isinstance(recognizer, CreditCardRecognizer) - assert len(analyzer_engine.analyze("My Credit card number is 4917300800000000", language="en")) > 0 + assert ( + len( + analyzer_engine.analyze( + "My Credit card number is 4917300800000000", language="en" + ) + ) + > 0 + ) def test_analyzer_engine_provider_no_nlp_recognizer_is_added(): @@ -344,7 +361,9 @@ def test_analyzer_engine_provider_no_nlp_recognizer_is_added_per_language(): analyzer_engine = provider.create_engine() - assert len(analyzer_engine.get_recognizers()) == 4 # Two CreditCardRecognizers and two SpacyRecognizers + assert ( + len(analyzer_engine.get_recognizers()) == 4 + ) # Two CreditCardRecognizers and two SpacyRecognizers nlp_recognizers = [ rec for rec in analyzer_engine.get_recognizers() @@ -372,7 +391,8 @@ def test_analyzer_engine_provider_multiple_nlp_recognizers_raises_exception(): with pytest.raises( ValueError, match=f"Multiple NLP recognizers for language en found in the configuration. " - f"Please remove the duplicates."): + f"Please remove the duplicates.", + ): provider = AnalyzerEngineProvider(analyzer_engine_conf_file=analyzer_yaml) analyzer_engine = provider.create_engine() @@ -385,7 +405,9 @@ def test_analyzer_engine_provider_no_nlp_engine_or_provider_results_in_default_n analyzer_engine = provider.create_engine() - assert len(analyzer_engine.get_recognizers()) == 2 # SpacyRecognizer, CreditCardRecognizer + assert ( + len(analyzer_engine.get_recognizers()) == 2 + ) # SpacyRecognizer, CreditCardRecognizer nlp_recognizer = [ rec for rec in analyzer_engine.get_recognizers() @@ -418,6 +440,7 @@ def test_analyzer_engine_stanza_without_recognizer_creates_recognizer(): } assert supported_languages == {"en", "es"} + def test_analyzer_engine_provider_one_custom_recognizer(): analyzer_yaml, _, _ = get_full_paths( "conf/custom_recognizer_yaml.yaml", @@ -426,7 +449,9 @@ def test_analyzer_engine_provider_one_custom_recognizer(): analyzer_engine = provider.create_engine() assert len(analyzer_engine.get_recognizers()) == 1 - assert analyzer_engine.analyze("My zip code is 12345", language="en")[0].score == pytest.approx(0.4) + assert analyzer_engine.analyze("My zip code is 12345", language="en")[ + 0 + ].score == pytest.approx(0.4) def test_analyzer_engine_provider_invalid_analyzer_conf_file(): @@ -444,7 +469,9 @@ def test_analyzer_engine_provider_invalid_nlp_conf_file(): def test_analyzer_engine_provider_invalid_registry_conf_file(): """Test that invalid recognizer registry configuration file path raises error.""" with pytest.raises(ValueError): - AnalyzerEngineProvider(recognizer_registry_conf_file="/nonexistent/path/file.yaml") + AnalyzerEngineProvider( + recognizer_registry_conf_file="/nonexistent/path/file.yaml" + ) def test_analyzer_engine_provider_get_configuration_with_nonexistent_file(): @@ -465,7 +492,7 @@ def test_analyzer_engine_provider_get_configuration_with_invalid_yaml(): import tempfile # Create a temporary file with invalid YAML - with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f: + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: f.write("invalid: yaml: content: [[[") temp_file = f.name @@ -703,8 +730,6 @@ def test_analyzer_engine_provider_configuration_logging(caplog): assert len(caplog.records) > 0 - - # --- install_models / _install_models_from_nlp_config tests --- _NLP_CONF_CONTENT = ( @@ -744,7 +769,9 @@ def test_install_models_analyzer_conf_without_nlp_falls_back_to_nlp_conf(tmp_pat nlp_yaml.write_text(_NLP_CONF_CONTENT) with patch("install_nlp_models._download_model") as mock_dl: - install_models(nlp_conf_file=str(nlp_yaml), analyzer_conf_file=str(analyzer_yaml)) + install_models( + nlp_conf_file=str(nlp_yaml), analyzer_conf_file=str(analyzer_yaml) + ) mock_dl.assert_called_once_with("spacy", "en_core_web_sm") @@ -768,7 +795,9 @@ def test_install_models_analyzer_conf_takes_priority_over_nlp_conf(tmp_path): nlp_yaml.write_text(_NLP_CONF_CONTENT) with patch("install_nlp_models._download_model") as mock_dl: - install_models(nlp_conf_file=str(nlp_yaml), analyzer_conf_file=str(analyzer_yaml)) + install_models( + nlp_conf_file=str(nlp_yaml), analyzer_conf_file=str(analyzer_yaml) + ) mock_dl.assert_called_once_with("spacy", "en_core_web_lg") diff --git a/presidio-analyzer/tests/test_app.py b/presidio-analyzer/tests/test_app.py new file mode 100644 index 0000000000..5c223d0f63 --- /dev/null +++ b/presidio-analyzer/tests/test_app.py @@ -0,0 +1,51 @@ +# ruff: noqa: D103 + +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from app import Server, _get_supported_languages + + +def test_get_supported_languages_from_environment(monkeypatch): + monkeypatch.setenv("SUPPORTED_LANGUAGES", " en, es ,it ") + + assert _get_supported_languages() == ["en", "es", "it"] + + +def test_get_supported_languages_is_unset(monkeypatch): + monkeypatch.delenv("SUPPORTED_LANGUAGES", raising=False) + + assert _get_supported_languages() is None + + +def test_server_passes_supported_languages_to_provider(monkeypatch): + monkeypatch.setenv("SUPPORTED_LANGUAGES", "en,es,it") + provider = Mock() + provider.return_value.create_engine.return_value = SimpleNamespace() + + with ( + patch("app.AnalyzerEngineProvider", provider), + patch("app.BatchAnalyzerEngine"), + ): + Server() + + assert provider.call_args.kwargs["supported_languages"] == ["en", "es", "it"] + + +def test_supported_entities_uses_configured_language(monkeypatch): + monkeypatch.setenv("SUPPORTED_LANGUAGES", "en,es") + engine = Mock() + engine.get_supported_entities.return_value = ["ES_NIF"] + provider = Mock() + provider.return_value.create_engine.return_value = engine + + with ( + patch("app.AnalyzerEngineProvider", provider), + patch("app.BatchAnalyzerEngine"), + ): + client = Server().app.test_client() + response = client.get("/supportedentities?language=es") + + assert response.status_code == 200 + assert response.get_json() == ["ES_NIF"] + engine.get_supported_entities.assert_called_once_with("es")