[formrecognizer] Migrate to test proxy (Azure#21822)

* use assert in testcase

* replace with assert is not none

* update remaining assertions

* initial test proxy work

* add ci.yml update

* temp

* temp

* test fixes

* change assert for error

* rstrip endpoint

* add sanitizers

* clean up replacers and FormRecognizerTest, move test vars

* use pytest.raises

* update test_business_card.py

* add business card recordings

* replace blob sas urls

* fix deleted receipt test

* update business card tests

* update content tests

* update custom forms tests

* update custom model tests

* update container url sanitizer and custom model import

* update general document tests

* update layout tests

* delete business card recordings

* remove all old recordings

* update dac prebuilt tests

* rstrip async client endpoint

* add oauth functions back in

* update compose tests

* skip copy model tests

* update dmac mgmt tests

* update dmac training tests

* update ftc compose tests

* update ftc copy model

* update conftest access token sanitizer

* update ftc mgmt tests

* update ftc training tests

* update get children tests

* update frc identity doc tests

* update frc invoice tests

* update frc receipt tests

* update multiapi tests

* clean up empty recordings

* remove extra empty recording

* update preparer values to match recordings

* dont check for body matching in requests that change during playback

* add playback condition in _get_blob_url

* default testing_container_sas_url

* set bodiless matcher for url tests

* set bodiless matcher for mgmt tests using DMAC and FTC

* move rstrip up in code

* remove unused code from conftest.py

* make large file tests run only live

* add missing imports

* remove extra space

* remove some more large recordings

* add missing recording

* add more recordings

Author: catalinaperalta

sdk/formrecognizer/azure-ai-formrecognizer/azure/ai/formrecognizer/_form_base_client.py

@@ -23,6 +23,11 @@
 class FormRecognizerClientBase(object):
     def __init__(self, endpoint, credential, **kwargs):
         # type: (str, Union[AzureKeyCredential, TokenCredential], Any) -> None
+        try:
+            endpoint = endpoint.rstrip("/")
+        except AttributeError:
+            raise ValueError("Parameter 'endpoint' must be a string.")
+
         self._endpoint = endpoint
         self._credential = credential
         self._api_version = kwargs.pop("api_version", None)

sdk/formrecognizer/azure-ai-formrecognizer/azure/ai/formrecognizer/aio/_form_base_client_async.py

@@ -34,6 +34,12 @@ def __init__(
         credential: Union["AzureKeyCredential", "AsyncTokenCredential"],
         **kwargs: Any
     ) -> None:
+
+        try:
+            endpoint = endpoint.rstrip("/")
+        except AttributeError:
+            raise ValueError("Parameter 'endpoint' must be a string.")
+
         self._endpoint = endpoint
         self._credential = credential
         self._api_version = kwargs.pop("api_version", None)
@@ -74,6 +80,7 @@ def __init__(
                 "api-version",
             }
         )
+
         self._client = FormRecognizer(
             endpoint=endpoint,
             credential=credential,  # type: ignore

sdk/formrecognizer/azure-ai-formrecognizer/tests/conftest.py

@@ -6,9 +6,46 @@
 # license information.
 # --------------------------------------------------------------------------
 
+import pytest
 import sys
+from devtools_testutils import add_remove_header_sanitizer, add_general_regex_sanitizer, add_oauth_response_sanitizer, add_body_key_sanitizer, test_proxy
 
 # Ignore async tests for Python < 3.6
 collect_ignore_glob = []
 if sys.version_info < (3, 6):
     collect_ignore_glob.append("*_async.py")
+
+@pytest.fixture(scope="session", autouse=True)
+def add_sanitizers(test_proxy):
+    add_remove_header_sanitizer(headers="Ocp-Apim-Subscription-Key")
+    add_general_regex_sanitizer(
+        value="fakeendpoint",
+        regex="(?<=\\/\\/)[a-z-]+(?=\\.cognitiveservices\\.azure\\.com)"
+    )
+    add_oauth_response_sanitizer()
+    add_body_key_sanitizer(
+        json_path="urlSource",
+        value="blob_sas_url",
+        regex="(?<=\\/\\/)[a-z-]+(?=\\.blob\\.core\\.windows\\.net)(.*)$",
+    )
+    add_body_key_sanitizer(
+        json_path="azureBlobSource.containerUrl",
+        value="blob_sas_url",
+        regex="(?<=\\/\\/)[a-z-]+(?=\\.blob\\.core\\.windows\\.net)(.*)$",
+    )
+    add_body_key_sanitizer(
+        json_path="source",
+        value="blob_sas_url",
+        regex="(?<=\\/\\/)[a-z-]+(?=\\.blob\\.core\\.windows\\.net)(.*)$",
+    )
+    add_body_key_sanitizer(
+        json_path="accessToken",
+        value="redacted",
+        regex="([0-9a-f-]{36})",
+    )
+    
+    add_body_key_sanitizer(
+        json_path="copyAuthorization.accessToken",
+        value="redacted",
+        regex="([0-9a-f-]{36})",
+    )

sdk/formrecognizer/azure-ai-formrecognizer/tests/preparers.py

@@ -18,21 +18,21 @@
 FormRecognizerPreparer = functools.partial(
     PowerShellPreparer,
     'formrecognizer',
-    formrecognizer_test_endpoint="https://region.api.cognitive.microsoft.com/",
+    formrecognizer_test_endpoint="https://fakeendpoint.cognitiveservices.azure.com",
     formrecognizer_test_api_key="fakeZmFrZV9hY29jdW50X2tleQ==",
-    formrecognizer_storage_container_sas_url="container_sas_url",
-    formrecognizer_testing_data_container_sas_url="container_sas_url",
-    formrecognizer_multipage_storage_container_sas_url="container_sas_url",
-    formrecognizer_multipage_storage_container_sas_url_2="container_sas_url",
-    formrecognizer_selection_mark_storage_container_sas_url="container_sas_url",
-    formrecognizer_table_variable_rows_container_sas_url="container_sas_url",
-    formrecognizer_table_fixed_rows_container_sas_url="container_sas_url",
-    formrecognizer_storage_container_sas_url_v2="container_sas_url",
-    formrecognizer_multipage_storage_container_sas_url_v2="container_sas_url",
-    formrecognizer_multipage_storage_container_sas_url_2_v2="container_sas_url",
-    formrecognizer_selection_mark_storage_container_sas_url_v2="container_sas_url",
-    formrecognizer_table_variable_rows_container_sas_url_v2="container_sas_url",
-    formrecognizer_table_fixed_rows_container_sas_url_v2="container_sas_url",
+    formrecognizer_storage_container_sas_url="https://blob_sas_url",
+    formrecognizer_testing_data_container_sas_url="https://blob_sas_url",
+    formrecognizer_multipage_storage_container_sas_url="https://blob_sas_url",
+    formrecognizer_multipage_storage_container_sas_url_2="https://blob_sas_url",
+    formrecognizer_selection_mark_storage_container_sas_url="https://blob_sas_url",
+    formrecognizer_table_variable_rows_container_sas_url="https://blob_sas_url",
+    formrecognizer_table_fixed_rows_container_sas_url="https://blob_sas_url",
+    formrecognizer_storage_container_sas_url_v2="https://blob_sas_url",
+    formrecognizer_multipage_storage_container_sas_url_v2="https://blob_sas_url",
+    formrecognizer_multipage_storage_container_sas_url_2_v2="https://blob_sas_url",
+    formrecognizer_selection_mark_storage_container_sas_url_v2="https://blob_sas_url",
+    formrecognizer_table_variable_rows_container_sas_url_v2="https://blob_sas_url",
+    formrecognizer_table_fixed_rows_container_sas_url_v2="https://blob_sas_url",
     formrecognizer_resource_id="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rgname/providers/Microsoft.CognitiveServices/accounts/frname",
     formrecognizer_region="region"
 )
@@ -53,7 +53,7 @@ def create_resource(self, name, **kwargs):
             form_recognizer_account_key = os.environ["FORMRECOGNIZER_TEST_API_KEY"]
             polling_interval = 5
         else:
-            form_recognizer_account = "https://region.api.cognitive.microsoft.com/"
+            form_recognizer_account = "https://fakeendpoint.cognitiveservices.azure.com"
             form_recognizer_account_key = "fakeZmFrZV9hY29jdW50X2tleQ=="
             polling_interval = 0