From 8a80f3af2b9ba9d894bd9a00450bcb2e79a38424 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Fri, 21 Feb 2025 17:27:58 -0800 Subject: [PATCH 1/2] Fix broken vectorstore tests for pdfs --- tests/mlmodel_langchain/test_vectorstore.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/mlmodel_langchain/test_vectorstore.py b/tests/mlmodel_langchain/test_vectorstore.py index 74275120a..050b3e54f 100644 --- a/tests/mlmodel_langchain/test_vectorstore.py +++ b/tests/mlmodel_langchain/test_vectorstore.py @@ -81,9 +81,13 @@ def vectorstore_events_sans_content(event): "id": None, # UUID that changes with each run "vendor": "langchain", "ingest_source": "Python", - "metadata.source": os.path.join(os.path.dirname(__file__), "hello.pdf"), - "metadata.page": 0, + "metadata.creationdate": "2023-02-10T00:49:47+00:00", + "metadata.creator": "XeTeX output 2023.02.10:0049", "metadata.page_label": "1", + "metadata.page": 0, + "metadata.producer": "xdvipdfmx (20210318)", + "metadata.source": os.path.join(os.path.dirname(__file__), "hello.pdf"), + "metadata.total_pages": 1, }, ), ] From a0da684d849ec87f50246a464521699e0f386901 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Fri, 21 Feb 2025 17:28:09 -0800 Subject: [PATCH 2/2] Format --- tests/mlmodel_langchain/test_vectorstore.py | 54 ++++++--------------- 1 file changed, 15 insertions(+), 39 deletions(-) diff --git a/tests/mlmodel_langchain/test_vectorstore.py b/tests/mlmodel_langchain/test_vectorstore.py index 050b3e54f..f95d70590 100644 --- a/tests/mlmodel_langchain/test_vectorstore.py +++ b/tests/mlmodel_langchain/test_vectorstore.py @@ -28,12 +28,8 @@ ) from testing_support.validators.validate_custom_event import validate_custom_event_count from testing_support.validators.validate_custom_events import validate_custom_events -from testing_support.validators.validate_error_trace_attributes import ( - validate_error_trace_attributes, -) -from testing_support.validators.validate_transaction_metrics import ( - validate_transaction_metrics, -) +from testing_support.validators.validate_error_trace_attributes import validate_error_trace_attributes +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics from newrelic.api.background_task import background_task from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes @@ -137,9 +133,7 @@ def test_vectorstore_modules_instrumented(): name="test_vectorstore:test_pdf_pagesplitter_vectorstore_in_txn", scoped_metrics=[("Llm/vectorstore/LangChain/similarity_search", 1)], rollup_metrics=[("Llm/vectorstore/LangChain/similarity_search", 1)], - custom_metrics=[ - (f"Supportability/Python/ML/LangChain/{langchain.__version__}", 1), - ], + custom_metrics=[(f"Supportability/Python/ML/LangChain/{langchain.__version__}", 1)], background_task=True, ) @validate_attributes("agent", ["llm"]) @@ -169,9 +163,7 @@ def test_pdf_pagesplitter_vectorstore_in_txn(set_trace_info, embedding_openai_cl name="test_vectorstore:test_pdf_pagesplitter_vectorstore_in_txn_no_content", scoped_metrics=[("Llm/vectorstore/LangChain/similarity_search", 1)], rollup_metrics=[("Llm/vectorstore/LangChain/similarity_search", 1)], - custom_metrics=[ - (f"Supportability/Python/ML/LangChain/{langchain.__version__}", 1), - ], + custom_metrics=[(f"Supportability/Python/ML/LangChain/{langchain.__version__}", 1)], background_task=True, ) @validate_attributes("agent", ["llm"]) @@ -229,9 +221,7 @@ def test_pdf_pagesplitter_vectorstore_ai_monitoring_disabled(set_trace_info, emb name="test_vectorstore:test_async_pdf_pagesplitter_vectorstore_in_txn", scoped_metrics=[("Llm/vectorstore/LangChain/asimilarity_search", 1)], rollup_metrics=[("Llm/vectorstore/LangChain/asimilarity_search", 1)], - custom_metrics=[ - (f"Supportability/Python/ML/LangChain/{langchain.__version__}", 1), - ], + custom_metrics=[(f"Supportability/Python/ML/LangChain/{langchain.__version__}", 1)], background_task=True, ) @validate_attributes("agent", ["llm"]) @@ -265,9 +255,7 @@ async def _test(): name="test_vectorstore:test_async_pdf_pagesplitter_vectorstore_in_txn_no_content", scoped_metrics=[("Llm/vectorstore/LangChain/asimilarity_search", 1)], rollup_metrics=[("Llm/vectorstore/LangChain/asimilarity_search", 1)], - custom_metrics=[ - (f"Supportability/Python/ML/LangChain/{langchain.__version__}", 1), - ], + custom_metrics=[(f"Supportability/Python/ML/LangChain/{langchain.__version__}", 1)], background_task=True, ) @validate_attributes("agent", ["llm"]) @@ -341,23 +329,20 @@ async def _test(): "ingest_source": "Python", "error": True, }, - ), + ) ] @reset_core_stats_engine() @validate_error_trace_attributes( - callable_name(AssertionError), - required_params={"user": ["vector_store_id"], "intrinsic": [], "agent": []}, + callable_name(AssertionError), required_params={"user": ["vector_store_id"], "intrinsic": [], "agent": []} ) @validate_custom_events(events_with_context_attrs(vectorstore_error_events)) @validate_transaction_metrics( name="test_vectorstore:test_vectorstore_error", scoped_metrics=[("Llm/vectorstore/LangChain/similarity_search", 1)], rollup_metrics=[("Llm/vectorstore/LangChain/similarity_search", 1)], - custom_metrics=[ - (f"Supportability/Python/ML/LangChain/{langchain.__version__}", 1), - ], + custom_metrics=[(f"Supportability/Python/ML/LangChain/{langchain.__version__}", 1)], background_task=True, ) @background_task() @@ -376,17 +361,14 @@ def test_vectorstore_error(set_trace_info, embedding_openai_client, loop): @reset_core_stats_engine() @disabled_ai_monitoring_record_content_settings @validate_error_trace_attributes( - callable_name(AssertionError), - required_params={"user": ["vector_store_id"], "intrinsic": [], "agent": []}, + callable_name(AssertionError), required_params={"user": ["vector_store_id"], "intrinsic": [], "agent": []} ) @validate_custom_events(vectorstore_events_sans_content(vectorstore_error_events)) @validate_transaction_metrics( name="test_vectorstore:test_vectorstore_error_no_content", scoped_metrics=[("Llm/vectorstore/LangChain/similarity_search", 1)], rollup_metrics=[("Llm/vectorstore/LangChain/similarity_search", 1)], - custom_metrics=[ - (f"Supportability/Python/ML/LangChain/{langchain.__version__}", 1), - ], + custom_metrics=[(f"Supportability/Python/ML/LangChain/{langchain.__version__}", 1)], background_task=True, ) @background_task() @@ -403,17 +385,14 @@ def test_vectorstore_error_no_content(set_trace_info, embedding_openai_client): @reset_core_stats_engine() @validate_error_trace_attributes( - callable_name(AssertionError), - required_params={"user": ["vector_store_id"], "intrinsic": [], "agent": []}, + callable_name(AssertionError), required_params={"user": ["vector_store_id"], "intrinsic": [], "agent": []} ) @validate_custom_events(events_with_context_attrs(vectorstore_error_events)) @validate_transaction_metrics( name="test_vectorstore:test_async_vectorstore_error", scoped_metrics=[("Llm/vectorstore/LangChain/asimilarity_search", 1)], rollup_metrics=[("Llm/vectorstore/LangChain/asimilarity_search", 1)], - custom_metrics=[ - (f"Supportability/Python/ML/LangChain/{langchain.__version__}", 1), - ], + custom_metrics=[(f"Supportability/Python/ML/LangChain/{langchain.__version__}", 1)], background_task=True, ) @background_task() @@ -437,17 +416,14 @@ async def _test(): @reset_core_stats_engine() @disabled_ai_monitoring_record_content_settings @validate_error_trace_attributes( - callable_name(AssertionError), - required_params={"user": ["vector_store_id"], "intrinsic": [], "agent": []}, + callable_name(AssertionError), required_params={"user": ["vector_store_id"], "intrinsic": [], "agent": []} ) @validate_custom_events(vectorstore_events_sans_content(vectorstore_error_events)) @validate_transaction_metrics( name="test_vectorstore:test_async_vectorstore_error_no_content", scoped_metrics=[("Llm/vectorstore/LangChain/asimilarity_search", 1)], rollup_metrics=[("Llm/vectorstore/LangChain/asimilarity_search", 1)], - custom_metrics=[ - (f"Supportability/Python/ML/LangChain/{langchain.__version__}", 1), - ], + custom_metrics=[(f"Supportability/Python/ML/LangChain/{langchain.__version__}", 1)], background_task=True, ) @background_task()