Deploy log analytics in same resource group as workspace in basic scenario (Azure#29304)

* deploy log analytics with workspace in same rg

* fix casing

* disable cspell for single line

* fix test

* remove space

---------

Co-authored-by: Josh Harrington <[email protected]>

Author: joshharrin

sdk/ml/azure-ai-ml/azure/ai/ml/_arm_deployments/arm_deployment_executor.py

@@ -168,6 +168,12 @@ def _check_deployment_status(self) -> None:
             resource_name = (
                 f"{arm_id_obj.asset_name} {arm_id_obj.asset_version if hasattr(arm_id_obj,'asset_version') else ''}"
             )
+            # do swap on asset_type to avoid collision with workspaces asset_type in arm id
+            arm_id_obj.asset_type = (
+                arm_id_obj.asset_type
+                if not arm_id_obj.provider_namespace_with_type == "OperationalInsightsworkspaces"
+                else "LogAnalytics"
+            )
             deployment_message = deployment_message_mapping[arm_id_obj.asset_type].format(f"{resource_name} ")
             if target_resource.resource_name not in self._resources_being_deployed.keys():
                 self._resources_being_deployed[target_resource.resource_name] = (

sdk/ml/azure-ai-ml/azure/ai/ml/_arm_deployments/arm_helper.py

@@ -37,11 +37,12 @@
     ArmConstants.ONLINE_DEPLOYMENT_TYPE: "Creating or updating deployment: {0}",
     AzureMLResourceType.DEPLOYMENT: "Creating or updating deployment: {0}",
     ArmConstants.UPDATE_ONLINE_ENDPOINT_TYPE: "Updating traffic",
-    ArmConstants.KEY_VAULT_PARAMETER_NAME: "Creating KeyVault: ({0})",
-    ArmConstants.APP_INSIGHTS_PARAMETER_NAME: "Creating AppInsights: ({0})",
+    ArmConstants.KEY_VAULT_PARAMETER_NAME: "Creating Key Vault: ({0})",
+    ArmConstants.LOG_ANALYTICS: "Creating Log Analytics Workspace: ({0})",
+    ArmConstants.APP_INSIGHTS_PARAMETER_NAME: "Creating Application Insights: ({0})",
     ArmConstants.CONTAINER_REGISTRY_PARAMETER_NAME: "Creating Container Registry: ({0})",
     ArmConstants.STORAGE_ACCOUNT_PARAMETER_NAME: "Creating Storage Account: ({0})",
-    AzureMLResourceType.WORKSPACE: "Creating workspace: ({0})",
+    AzureMLResourceType.WORKSPACE: "Creating AzureML Workspace: ({0})",
     AzureMLResourceType.CONNECTIONS: "Creating connection: ({0})",
 }
 

sdk/ml/azure-ai-ml/azure/ai/ml/_arm_deployments/arm_templates/workspace_base.json

@@ -128,6 +128,20 @@
                 "description": "Determines whether or not new ApplicationInsights should be provisioned."
             }
         },
+        "logAnalyticsName": {
+            "type": "string",
+            "defaultValue": "[concat('la', uniqueString(parameters('resourceGroupName'), parameters('workspaceName')))]",
+            "metadata": {
+                "description": "Name of LogAnalytics to be used by ApplicationInsights resource."
+            }
+        },
+        "logAnalyticsArmId": {
+            "type": "string",
+            "defaultValue": "",
+            "metadata": {
+                "description": "ARM Id of LogAnalytics to be used by ApplicationInsights resource."
+            }
+        },
         "applicationInsightsName": {
             "type": "string",
             "defaultValue": "[concat('ai', uniqueString(parameters('resourceGroupName'), parameters('workspaceName')))]",
@@ -597,6 +611,18 @@
                 "networkRuleSet": "[if(equals(parameters('containerRegistryBehindVNet'), 'true'), variables('networkRuleSetBehindVNet'), json('null'))]"
             }
         },
+        {
+            "condition": "[and(variables('enablePE'), equals(parameters('applicationInsightsOption'), 'new'))]",
+            "type": "Microsoft.OperationalInsights/workspaces",
+            "tags": "[parameters('tagValues')]",
+            "apiVersion": "2020-08-01",
+            "name": "[parameters('logAnalyticsName')]",
+            "location": "[if(or(equals(toLower(parameters('applicationInsightsLocation')),'westcentralus'), equals(toLower(parameters('applicationInsightsLocation')),'eastus2euap'), equals(toLower(parameters('applicationInsightsLocation')),'centraluseuap')),'southcentralus', parameters('applicationInsightsLocation'))]",
+            "kind": "web",
+            "properties": {
+                "Application_Type": "web"
+            }
+        },
         {
             "condition": "[and(variables('enablePE'), equals(parameters('applicationInsightsOption'), 'new'))]",
             "type": "Microsoft.Insights/components",
@@ -605,8 +631,12 @@
             "name": "[parameters('applicationInsightsName')]",
             "location": "[if(or(equals(toLower(parameters('applicationInsightsLocation')),'westcentralus'), equals(toLower(parameters('applicationInsightsLocation')),'eastus2euap'), equals(toLower(parameters('applicationInsightsLocation')),'centraluseuap')),'southcentralus', parameters('applicationInsightsLocation'))]",
             "kind": "web",
+            "dependsOn": [
+                "[resourceId('Microsoft.OperationalInsights/workspaces', parameters('logAnalyticsName'))]"
+            ],
             "properties": {
-                "Application_Type": "web"
+                "Application_Type": "web",
+                "WorkspaceResourceId": "[parameters('logAnalyticsArmId')]"
             }
         },
         {

sdk/ml/azure-ai-ml/azure/ai/ml/_arm_deployments/arm_templates/workspace_param.json

@@ -44,6 +44,12 @@
     "applicationInsightsName": {
         "value": ""
     },
+    "logAnalyticsName": {
+        "value": ""
+    },
+    "logAnalyticsArmId": {
+        "value": ""
+    },
     "applicationInsightsResourceGroupName": {
         "value": ""
     },

sdk/ml/azure-ai-ml/azure/ai/ml/_utils/_appinsights_utils.py

@@ -4,114 +4,13 @@
 
 import logging
 
-from azure.ai.ml._azure_environments import _get_base_url_from_metadata
-from azure.ai.ml._vendor.azure_resources._resource_management_client import ResourceManagementClient
-from azure.ai.ml.constants._common import ArmConstants
-from azure.core.credentials import TokenCredential
-from azure.core.exceptions import HttpResponseError
-from azure.ai.ml.constants._workspace import AppInsightsDefaults
-
 module_logger = logging.getLogger(__name__)
 
 
-def default_resource_group_for_app_insights_exists(
-    credentials: TokenCredential, subscription_id: str, location: str
-) -> bool:
-    client = ResourceManagementClient(
-        credential=credentials,
-        subscription_id=subscription_id,
-        base_url=_get_base_url_from_metadata(),
-        api_version=ArmConstants.AZURE_MGMT_RESOURCE_API_VERSION,
-    )
-    default_rgName = AppInsightsDefaults.DEFAULT_RESOURCE_GROUP_NAME.format(location=location)
-    try:
-        client.resource_groups.get(resource_group_name=default_rgName)
-        return True
-    except HttpResponseError:
-        return False
-
-
-def default_log_analytics_workspace_exists(credentials: TokenCredential, subscription_id: str, location: str) -> bool:
-    client = ResourceManagementClient(
-        credential=credentials,
-        subscription_id=subscription_id,
-        base_url=_get_base_url_from_metadata(),
-        api_version=ArmConstants.AZURE_MGMT_RESOURCE_API_VERSION,
-    )
-    default_resource_group = AppInsightsDefaults.DEFAULT_RESOURCE_GROUP_NAME.format(location=location)
-    default_workspace = client.resources.list_by_resource_group(
-        default_resource_group,
-        filter=f"substringof('{AppInsightsDefaults.DEFAULT_LOG_ANALYTICS_NAME.format(location=location)}',name)",
-    )
-    for item in default_workspace:  # pylint: disable=unused-variable
-        # return true for is_existing
-        return True
-    # else return false for is_existing
-    return False
-
-
-def get_default_log_analytics_arm_id(subscription_id: str, location: str) -> str:
+def get_log_analytics_arm_id(subscription_id: str, resource_group_name: str, log_analytics_name: str) -> str:
     return (
         f"/subscriptions/{subscription_id}/"
-        f"resourceGroups/{AppInsightsDefaults.DEFAULT_RESOURCE_GROUP_NAME.format(location=location)}/"
+        f"resourceGroups/{resource_group_name}/"
         "providers/Microsoft.OperationalInsights/workspaces/"
-        f"{AppInsightsDefaults.DEFAULT_LOG_ANALYTICS_NAME.format(location=location)}"
+        f"{log_analytics_name}"
     )
-
-
-def get_default_resource_group_deployment(deployment_name: str, location: str, subscription_id: str) -> dict:
-    return {
-        "type": "Microsoft.Resources/deployments",
-        "apiVersion": "2019-10-01",
-        "name": deployment_name,
-        "subscriptionId": subscription_id,
-        "location": location,
-        "properties": {
-            "mode": "Incremental",
-            "template": {
-                "$schema": (
-                    "https://schema.management.azure.com/schemas/2018-05-01/" + "subscriptionDeploymentTemplate.json#"
-                ),
-                "contentVersion": "1.0.0.1",
-                "parameters": {},
-                "variables": {},
-                "resources": [
-                    {
-                        "type": "Microsoft.Resources/resourceGroups",
-                        "apiVersion": "2018-05-01",
-                        "location": location,
-                        "name": AppInsightsDefaults.DEFAULT_RESOURCE_GROUP_NAME.format(location=location),
-                        "properties": {},
-                    }
-                ],
-            },
-        },
-    }
-
-
-def get_default_log_analytics_deployment(deployment_name: str, location: str, subscription_id: str) -> dict:
-    return {
-        "type": "Microsoft.Resources/deployments",
-        "apiVersion": "2019-10-01",
-        "name": deployment_name,
-        "resourceGroup": AppInsightsDefaults.DEFAULT_RESOURCE_GROUP_NAME.format(location=location),
-        "subscriptionId": subscription_id,
-        "properties": {
-            "mode": "Incremental",
-            "template": {
-                "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
-                "contentVersion": "1.0.0.0",
-                "parameters": {},
-                "variables": {},
-                "resources": [
-                    {
-                        "apiVersion": "2020-08-01",
-                        "name": AppInsightsDefaults.DEFAULT_LOG_ANALYTICS_NAME.format(location=location),
-                        "type": "Microsoft.OperationalInsights/workspaces",
-                        "location": location,
-                        "properties": {},
-                    }
-                ],
-            },
-        },
-    }

sdk/ml/azure-ai-ml/azure/ai/ml/_utils/_arm_id_utils.py

@@ -221,6 +221,7 @@ def __init__(self, arm_id=None):
             if match:
                 self.subscription_id = match.group(1)
                 self.resource_group_name = match.group(2)
+                self.provider_namespace_with_type = match.group(3) + match.group(4)
                 self.asset_type = match.group(4)
                 self.asset_name = match.group(5)
             elif rg_match:

sdk/ml/azure-ai-ml/azure/ai/ml/constants/_common.py

@@ -303,6 +303,7 @@ class ArmConstants:
     STORAGE = "StorageAccount"
     KEY_VAULT = "KeyVault"
     APP_INSIGHTS = "AppInsights"
+    LOG_ANALYTICS = "LogAnalytics"
     WORKSPACE = "Workspace"
 
     AZURE_MGMT_RESOURCE_API_VERSION = "2020-06-01"

sdk/ml/azure-ai-ml/azure/ai/ml/constants/_workspace.py

@@ -6,11 +6,6 @@
 from azure.core import CaseInsensitiveEnumMeta
 
 
-class AppInsightsDefaults(object):
-    DEFAULT_LOG_ANALYTICS_NAME = "DefaultWorkspace-{location}"
-    DEFAULT_RESOURCE_GROUP_NAME = "DefaultResourceGroup-{location}"
-
-
 class ManagedServiceIdentityType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """Type of managed service identity (where both SystemAssigned and UserAssigned types are allowed)."""
 

sdk/ml/azure-ai-ml/azure/ai/ml/operations/_workspace_operations_base.py

@@ -28,13 +28,7 @@
     get_resource_and_group_name,
     get_resource_group_location,
 )
-from azure.ai.ml._utils._appinsights_utils import (
-    default_resource_group_for_app_insights_exists,
-    default_log_analytics_workspace_exists,
-    get_default_resource_group_deployment,
-    get_default_log_analytics_deployment,
-    get_default_log_analytics_arm_id,
-)
+from azure.ai.ml._utils._appinsights_utils import get_log_analytics_arm_id
 from azure.ai.ml._utils.utils import camel_to_snake, from_iso_duration_format_min_sec
 from azure.ai.ml._version import VERSION
 from azure.ai.ml.constants import ManagedServiceIdentityType
@@ -360,62 +354,12 @@ def _populate_arm_paramaters(self, workspace: Workspace, **kwargs: Dict) -> Tupl
                 group_name,
             )
         else:
-            # if workspace is located in a region where app insights is not supported, we do this swap
-            if workspace.location in ["westcentralus", "eastus2euap", "centraluseuap"]:
-                app_insights_location = "southcentralus"
-            else:
-                app_insights_location = workspace.location
-            # check if default resource group already exists
-            rg_is_existing = default_resource_group_for_app_insights_exists(
-                self._credentials, self._subscription_id, app_insights_location
+            log_analytics = _generate_log_analytics(workspace.name, resources_being_deployed)
+            _set_val(param["logAnalyticsName"], log_analytics)
+            _set_val(
+                param["logAnalyticsArmId"],
+                get_log_analytics_arm_id(self._subscription_id, self._resource_group_name, log_analytics),
             )
-            # if default resource group does not exist yet, create rg and log analytics
-            if not rg_is_existing:
-                # add resource group and log analytics deployments to resources
-                deployment_string = get_deployment_name("")
-                app_insights_resource_group_deployment_name = f"DeployResourceGroup{deployment_string}"
-                app_insights_log_workspace_deployment_name = f"DeployLogWorkspace{deployment_string}"
-                template["resources"].append(
-                    get_default_resource_group_deployment(
-                        app_insights_resource_group_deployment_name, app_insights_location, self._subscription_id
-                    )
-                )
-                log_analytics_deployment = get_default_log_analytics_deployment(
-                    app_insights_log_workspace_deployment_name, app_insights_location, self._subscription_id
-                )
-                log_analytics_deployment["dependsOn"] = [app_insights_resource_group_deployment_name]
-                template["resources"].append(log_analytics_deployment)
-                for resource in template["resources"]:
-                    if resource["type"] == "Microsoft.Insights/components":
-                        resource["dependsOn"] = [app_insights_log_workspace_deployment_name]
-            # if default resource group exists, check default log analytics exists
-            else:
-                # check if default log analytics workspace already exists
-                la_is_existing = default_log_analytics_workspace_exists(
-                    self._credentials, self._subscription_id, app_insights_location
-                )
-                # if this does not exist yet, add the deployment needed to resources
-                if not la_is_existing:
-                    deployment_string = get_deployment_name("")
-                    app_insights_log_workspace_deployment_name = f"DeployLogWorkspace{deployment_string}"
-                    template["resources"].append(
-                        get_default_log_analytics_deployment(
-                            app_insights_log_workspace_deployment_name, app_insights_location, self._subscription_id
-                        )
-                    )
-                    for resource in template["resources"]:
-                        if resource["type"] == "Microsoft.Insights/components":
-                            resource["dependsOn"] = [app_insights_log_workspace_deployment_name]
-
-            # add WorkspaceResourceId property to app insights in template
-            for resource in template["resources"]:
-                if resource["type"] == "Microsoft.Insights/components":
-                    resource["properties"] = {
-                        "Application_Type": "web",
-                        "WorkspaceResourceId": get_default_log_analytics_arm_id(
-                            self._subscription_id, app_insights_location
-                        ),
-                    }
 
             app_insights = _generate_app_insights(workspace.name, resources_being_deployed)
             _set_val(param["applicationInsightsName"], app_insights)
@@ -543,6 +487,15 @@ def _generate_storage(name: str, resources_being_deployed: dict) -> str:
     return storage
 
 
+def _generate_log_analytics(name: str, resources_being_deployed: dict) -> str:
+    log_analytics = get_name_for_dependent_resource(name, "logalytics")  # cspell:disable-line
+    resources_being_deployed[log_analytics] = (
+        ArmConstants.LOG_ANALYTICS,
+        None,
+    )
+    return log_analytics
+
+
 def _generate_app_insights(name: str, resources_being_deployed: dict) -> str:
     # Application name only allows alphanumeric characters, periods, underscores,
     # hyphens and parenthesis and cannot end in a period

sdk/ml/azure-ai-ml/tests/workspace/unittests/test_workspace_operations_base.py

@@ -215,7 +215,7 @@ def test_populate_arm_paramaters(
             "azure.ai.ml.operations._workspace_operations_base.get_resource_group_location", return_value="random_name"
         )
         mocker.patch(
-            "azure.ai.ml.operations._workspace_operations_base.get_default_log_analytics_arm_id",
+            "azure.ai.ml.operations._workspace_operations_base.get_log_analytics_arm_id",
             return_value=("random_id", True),
         )
         mock_workspace_operation_base._populate_arm_paramaters(workspace=Workspace(name="name"))