From f3fabddf107ae9d27621aa630e356414b1452293 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 7 Nov 2024 07:57:17 -0600 Subject: [PATCH 01/87] Reworking Storage Secrets --- README.md | 4 +- bicep/main.bicep | 72 ++- bicep/modules/blade_common.bicep | 478 --------------- bicep/modules/blade_partition.bicep | 86 +-- bicep/modules/cosmos-db/main.bicep | 45 ++ .../Legal_COO.json | 0 .../blob_upload.sh} | 6 +- .../modules/deploy-scripts/software-upload.sh | 45 ++ bicep/modules/script-blob-upload/main.bicep | 94 --- bicep/modules/script-kv-certificate/README.md | 147 ----- .../modules/script-kv-certificate/main.bicep | 170 ------ bicep/modules/script-kv-certificate/script.sh | 71 --- .../test/main.test.bicep | 104 ---- .../test/parameters.json | 15 - .../script-kv-certificate/version.json | 7 - bicep/modules/script-share-csvdag/main.bicep | 2 +- bicep/modules/script-share-csvdag/script.sh | 2 +- bicep/modules/script-share-upload/main.bicep | 2 +- bicep/modules/script-share-upload/script.sh | 6 +- .../.bicep/keyvault_secrets.bicep | 31 - .../.bicep/nested_rbac.bicep | 70 --- bicep/modules/storage-account-orig/README.md | 128 ---- bicep/modules/storage-account-orig/main.bicep | 574 ------------------ .../storage-account-orig/metadata.json | 6 - .../storage-account-orig/test/main.test.bicep | 20 - .../storage-account-orig/test/parameters.json | 12 - .../modules/storage-account-orig/version.json | 8 - software/components/airflow/pvc.yaml | 24 +- tools/rest-scripts/partition.http | 4 +- 29 files changed, 210 insertions(+), 2023 deletions(-) delete mode 100644 bicep/modules/blade_common.bicep rename bicep/modules/{script-blob-upload => deploy-scripts}/Legal_COO.json (100%) rename bicep/modules/{script-blob-upload/script.sh => deploy-scripts/blob_upload.sh} (74%) create mode 100644 bicep/modules/deploy-scripts/software-upload.sh delete mode 100644 bicep/modules/script-blob-upload/main.bicep delete mode 100644 bicep/modules/script-kv-certificate/README.md delete mode 100644 bicep/modules/script-kv-certificate/main.bicep delete mode 100644 bicep/modules/script-kv-certificate/script.sh delete mode 100644 bicep/modules/script-kv-certificate/test/main.test.bicep delete mode 100644 bicep/modules/script-kv-certificate/test/parameters.json delete mode 100644 bicep/modules/script-kv-certificate/version.json delete mode 100644 bicep/modules/storage-account-orig/.bicep/keyvault_secrets.bicep delete mode 100644 bicep/modules/storage-account-orig/.bicep/nested_rbac.bicep delete mode 100644 bicep/modules/storage-account-orig/README.md delete mode 100644 bicep/modules/storage-account-orig/main.bicep delete mode 100644 bicep/modules/storage-account-orig/metadata.json delete mode 100644 bicep/modules/storage-account-orig/test/main.test.bicep delete mode 100644 bicep/modules/storage-account-orig/test/parameters.json delete mode 100644 bicep/modules/storage-account-orig/version.json diff --git a/README.md b/README.md index 80ae7382..bc6b731a 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ This project uses the following Azure Container Service preview features: - [Node Auto Provisioning](https://learn.microsoft.com/en-us/azure/aks/node-autoprovision?tabs=azure-cli) - [SSH Disable](https://learn.microsoft.com/en-us/azure/aks/manage-ssh-node-access?tabs=node-shell#disable-ssh-overview) -Review the [Getting Started](https://azure.github.io/osdu-developer/getting_started/) documentation prior to proceeding. +Review the [documentation](https://azure.github.io/osdu-developer/getting_started/) prior to proceeding. ### CLI Quickstart @@ -58,7 +58,7 @@ azd hooks run settings azd down --force --purge ``` -### Portal Quickstart _(reduced capabilities)_ +### Portal Quickstart > **Tutorial:** [Deploy OSDU Personal Instance via Portal](https://azure.github.io/osdu-developer/tutorial_arm/) diff --git a/bicep/main.bicep b/bicep/main.bicep index bd339048..717b64eb 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -367,7 +367,7 @@ module clusterBlade 'modules/blade_cluster.bicep' = { enableTelemetry: enableTelemetry enableNodeAutoProvisioning: clusterConfiguration.enableNodeAutoProvisioning == 'false' ? false : true - enablePrivateCluster: clusterConfiguration.enablePrivateCluster == 'false' ? false : true + enablePrivateCluster: clusterConfiguration.enablePrivateCluster == 'true' ? true : false workspaceResourceId: logAnalytics.outputs.resourceId identityId: enableVnetInjection ? networkBlade.outputs.networkConfiguration.identityId : stampIdentity.outputs.resourceId @@ -758,15 +758,17 @@ module storage 'modules/storage-account/main.bicep' = { principalId: stampIdentity.outputs.principalId principalType: 'ServicePrincipal' } - + { + roleDefinitionIdOrName: 'Storage File Data Privileged Contributor' + principalId: stampIdentity.outputs.principalId + principalType: 'ServicePrincipal' + } ] // Apply Security allowBlobPublicAccess: enableBlobPublicAccess publicNetworkAccess: 'Enabled' - - // TODO: Deployment Scripts don't support this yet. - // allowSharedKeyAccess: true + allowSharedKeyAccess: true // https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/deployment-script-template?tabs=CLI#debug-deployment-scripts networkAcls: { bypass: 'AzureServices' @@ -832,6 +834,16 @@ module database 'modules/cosmos-db/main.bicep' = { diagnosticWorkspaceId: logAnalytics.outputs.resourceId diagnosticLogsRetentionInDays: 0 + networkRestrictions: { + publicNetworkAccess: 'Enabled' + networkAclBypass: 'AzureServices' + ipRules: [ + '${clusterBlade.outputs.natClusterIP}' + ] + virtualNetworkRules: [] + } + + // Configure Service capabilitiesToAdd: [ 'EnableGremlin' @@ -898,22 +910,39 @@ var directoryUploads = [ ] @batchSize(1) -module gitOpsUpload 'modules/software-upload/main.bicep' = [for item in directoryUploads: { +module gitOpsUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for item in directoryUploads: { name: '${configuration.name}-storage-${item.directory}-upload' params: { - newStorageAccount: true + name: 'script-${storage.outputs.name}-${item.directory}' + location: location - storageAccountName: storage.outputs.name - identityName: stampIdentity.outputs.name + cleanupPreference: 'Always' + retentionInterval: 'PT1H' + timeout: 'PT30M' + runOnce: true + + managedIdentities: { + userAssignedResourcesIds: [ + stampIdentity.outputs.resourceId + ] + } - directoryName: item.directory + kind: 'AzureCLI' + azCliVersion: '2.63.0' + + environmentVariables: [ + { name: 'AZURE_STORAGE_ACCOUNT', value: storage.outputs.name } + { name: 'FILE', value: 'main.zip' } + { name: 'URL', value: 'https://github.com/azure/osdu-developer/archive/refs/heads/main.zip' } + { name: 'CONTAINER', value: 'gitops' } + { name: 'UPLOAD_DIR', value: string(item.directory) } + ] + scriptContent: loadTextContent('./modules/deploy-scripts/software-upload.sh') } - dependsOn: [ - stampIdentity - storage - ] }] + +//TODO: This needs to be removed and moved into a kubernetes job. module manifestDagShareUpload 'modules/script-share-upload/main.bicep' = { name: '${configuration.name}-storage-dag-upload-manifest' params: { @@ -933,6 +962,7 @@ module manifestDagShareUpload 'modules/script-share-upload/main.bicep' = { ] } +//TODO: This needs to be removed and moved into a kubernetes job. module csvDagShareUpload 'modules/script-share-csvdag/main.bicep' = { name: '${configuration.name}-storage-dag-upload-csv' params: { @@ -1079,6 +1109,20 @@ module configBlade 'modules/blade_configuration.bicep' = { ] } +module storageAcl 'modules/network_acl_storage.bicep' = { + name: '${configuration.name}-storage-acl' + params: { + storageName: storage.outputs.name + location: location + skuName: configuration.storage.sku + natClusterIP: clusterBlade.outputs.natClusterIP + } + dependsOn: [ + manifestDagShareUpload + csvDagShareUpload + gitOpsUpload + ] +} // =============== // // Outputs // diff --git a/bicep/modules/blade_common.bicep b/bicep/modules/blade_common.bicep deleted file mode 100644 index 9ae28bb5..00000000 --- a/bicep/modules/blade_common.bicep +++ /dev/null @@ -1,478 +0,0 @@ -///////////////// -// Common Blade -///////////////// - -@description('Optional. Indicates whether public access is enabled for all blobs or containers in the storage account. For security reasons, it is recommended to set it to false.') -param enableBlobPublicAccess bool - -@description('Optional. The tags to apply to the resources') -param tags object = {} - -@description('The location of resources to deploy') -param location string - -@description('Feature Flag to Enable Telemetry') -param enableTelemetry bool - -@description('The configuration for the blade section.') -param bladeConfig bladeSettings - -@description('The workspace resource Id for diagnostics') -param workspaceResourceId string - -@description('The Application Insights Instrumentation Key') -param insightsKey string - -@description('Conditional. The name of the parent user assigned identity. Required if the template is used in a standalone deployment.') -param identityName string - -@description('The IP address of the NAT cluster.') -param natClusterIP string - -@description('Optional. Customer Managed Encryption Key.') -param cmekConfiguration object = { - kvUrl: '' - keyName: '' - identityId: '' -} - -@description('Specify the AD Application Client Id.') -param applicationClientId string - -@description('Specify the AD Application Client Secret.') -@secure() -param applicationClientSecret string - -@description('Specify the AD Application Principal Id.') -param applicationClientPrincipalOid string = '' - - -var commonLayerConfig = { - insights: { - sku: 'web' - } - storage: { - sku: 'Standard_LRS' - containers: [ - 'system' - 'azure-webjobs-hosts' - 'azure-webjobs-eventhub' - 'gitops' - ] - tables: [ - 'partitionInfo' - ] - shares: [ - 'airflow-logs' - 'airflow-dags' - ] - } - database: { - name: 'osdu-graph' - throughput: 2000 - backup: 'Continuous' - graphs: [ - { - name: 'Entitlements' - automaticIndexing: true - partitionKeyPaths: [ - '/dataPartitionId' - ] - } - ] - } -} - -resource userAssignedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = { - name: identityName -} - -/* - __ ___ ___________ ____ ____ ____ ___ __ __ __ .___________. -| |/ / | ____\ \ / / \ \ / / / \ | | | | | | | | -| ' / | |__ \ \/ / \ \/ / / ^ \ | | | | | | `---| |----` -| < | __| \_ _/ \ / / /_\ \ | | | | | | | | -| . \ | |____ | | \ / / _____ \ | `--' | | `----. | | -|__|\__\ |_______| |__| \__/ /__/ \__\ \______/ |_______| |__| -*/ - -// var name = '${replace(bladeConfig.sectionName, '-', '')}${uniqueString(resourceGroup().id, bladeConfig.sectionName)}' - -// @description('The list of secrets to persist to the Key Vault') -// var vaultSecrets = [ -// { -// secretName: 'tenant-id' -// secretValue: subscription().tenantId -// } -// { -// secretName: 'app-dev-sp-tenant-id' -// secretValue: subscription().tenantId -// } -// { -// secretName: 'subscription-id' -// secretValue: subscription().subscriptionId -// } -// // Azure AD Secrets -// { -// secretName: 'app-dev-sp-password' -// secretValue: applicationClientSecret == '' ? 'dummy' : applicationClientSecret -// } -// { -// secretName: 'app-dev-sp-id' -// secretValue: applicationClientId -// } -// { -// secretName: 'cpng-user-name' -// secretValue: 'dbuser' -// } -// { -// secretName: 'cpng-user-password' -// secretValue: substring(uniqueString('dbuser', resourceGroup().id, bladeConfig.sectionName), 0, 8) -// } -// { -// secretName: 'cpng-superuser-name' -// secretValue: 'dbadmin' -// } -// { -// secretName: 'cpng-superuser-password' -// secretValue: substring(uniqueString('dbadmin', resourceGroup().id, bladeConfig.sectionName), 0, 8) -// } -// { -// secretName: 'airflow-db-connection' -// secretValue: 'postgresql://dbuser:${substring(uniqueString('dbuser', resourceGroup().id, bladeConfig.sectionName), 0, 8)}@airflow-cluster-rw.postgresql.svc.cluster.local:5432/airflow-db' -// } -// { -// secretName: 'airflow-admin-username' -// secretValue: 'admin' -// } -// { -// secretName: 'airflow-admin-password' -// secretValue: substring(uniqueString('airflow', resourceGroup().id, bladeConfig.sectionName), 0, 8) -// } -// { -// secretName: 'airflow-fernet-key' -// secretValue: substring(uniqueString('airflow-fernet', resourceGroup().id, bladeConfig.sectionName), 0, 8) -// } -// { -// secretName: 'airflow-webserver-key' -// secretValue: substring(uniqueString('airflow-webserver', resourceGroup().id, bladeConfig.sectionName), 0, 8) -// } -// ] - -// module keyvault 'br/public:avm/res/key-vault/vault:0.5.1' = { -// name: '${bladeConfig.sectionName}-keyvault' -// params: { -// name: length(name) > 24 ? substring(name, 0, 24) : name -// location: location -// enableTelemetry: enableTelemetry - -// // Assign Tags -// tags: union( -// tags, -// { -// layer: bladeConfig.displayName -// } -// ) - -// diagnosticSettings: [ -// { -// workspaceResourceId: workspaceResourceId -// } -// ] - -// enablePurgeProtection: false - -// // Configure RBAC -// enableRbacAuthorization: true -// roleAssignments: union( -// applicationClientPrincipalOid != '' ? [ -// { -// roleDefinitionIdOrName: 'Key Vault Secrets User' -// principalId: applicationClientPrincipalOid -// principalType: 'ServicePrincipal' -// } -// ] : [], -// [] -// ) - -// // Configure Network Access -// publicNetworkAccess: 'Enabled' -// networkAcls: { -// bypass: 'AzureServices' -// defaultAction: 'Deny' -// ipRules: [ -// { -// value: natClusterIP -// } -// ] -// } - -// // Configure Secrets -// secrets: { -// secureList: [for secret in vaultSecrets: { -// name: secret.secretName -// value: secret.secretValue -// }] -// } -// } -// } - - -/* _______.___________. ______ .______ ___ _______ _______ - / | | / __ \ | _ \ / \ / _____|| ____| - | (----`---| |----`| | | | | |_) | / ^ \ | | __ | |__ - \ \ | | | | | | | / / /_\ \ | | |_ | | __| -.----) | | | | `--' | | |\ \----./ _____ \ | |__| | | |____ -|_______/ |__| \______/ | _| `._____/__/ \__\ \______| |_______| -*/ -// AVM Module Customized due to required Secrets. -// module storage './storage-account/main.bicep' = { -// name: '${bladeConfig.sectionName}-storage' -// params: { -// name: '${replace(bladeConfig.sectionName, '-', '')}${uniqueString(resourceGroup().id, bladeConfig.sectionName)}' -// location: location -// skuName: commonLayerConfig.storage.sku - -// // Assign Tags -// tags: union( -// tags, -// { -// layer: bladeConfig.displayName -// } -// ) - -// // Hook up Diagnostics -// diagnosticSettings: [ -// { -// workspaceResourceId: workspaceResourceId -// } -// ] - -// // Configure Service -// blobServices: { -// containers: map(commonLayerConfig.storage.containers, container => { -// name: container -// }) -// } -// tableServices: { -// tables: map(commonLayerConfig.storage.tables, table => { -// name: table -// }) -// } -// fileServices: { -// shares: map(commonLayerConfig.storage.shares, share => { -// name: share -// }) -// } - -// // Apply Security -// allowBlobPublicAccess: enableBlobPublicAccess - -// publicNetworkAccess: 'Enabled' - -// // TODO: Deployment Scripts don't support this yet. -// // allowSharedKeyAccess: true -// // https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/deployment-script-template?tabs=CLI#debug-deployment-scripts -// networkAcls: { -// bypass: 'AzureServices' -// defaultAction: 'Allow' // <--- Allow all traffic. Should be changed to Deny. -// ipRules: [ -// { -// value: natClusterIP -// } -// ] -// } - -// // Persist Secrets to Vault -// secretsExportConfiguration: { -// keyVaultResourceId: keyvault.outputs.resourceId -// accountName: [ -// 'system-storage' -// 'tbl-storage' -// ] -// accessKey1: [ -// 'system-storage-key' -// 'tbl-storage-key' -// ] -// connectionString1: [ -// 'system-storage-connection' -// ] -// blobEndpoint: [ -// 'system-storage-blob-endpoint' -// ] -// tableEndpoint: [ -// 'tbl-storage-endpoint' -// ] -// } -// } -// } - - -/* - _______ .______ ___ .______ __ __ - / _____|| _ \ / \ | _ \ | | | | -| | __ | |_) | / ^ \ | |_) | | |__| | -| | |_ | | / / /_\ \ | ___/ | __ | -| |__| | | |\ \----./ _____ \ | | | | | | - \______| | _| `._____/__/ \__\ | _| |__| |__| -*/ -// AVM Module Customized due to required Secrets. -// module database './cosmos-db/main.bicep' = { -// name: '${bladeConfig.sectionName}-cosmos-db' -// params: { -// resourceName: bladeConfig.sectionName -// resourceLocation: location - -// // Assign Tags -// tags: union( -// tags, -// { -// layer: bladeConfig.displayName -// } -// ) - -// // Hook up Diagnostics -// diagnosticWorkspaceId: workspaceResourceId -// diagnosticLogsRetentionInDays: 0 - -// // Configure Service -// capabilitiesToAdd: [ -// 'EnableGremlin' -// ] -// gremlinDatabases: [ -// { -// name: commonLayerConfig.database.name -// graphs: commonLayerConfig.database.graphs -// } -// ] -// throughput: commonLayerConfig.database.throughput -// backupPolicyType: commonLayerConfig.database.backup - -// // Hookup Customer Managed Encryption Key -// systemAssignedIdentity: false -// userAssignedIdentities: !empty(cmekConfiguration.identityId) ? { -// '${cmekConfiguration.identityId}': {} -// } : {} -// defaultIdentity: !empty(cmekConfiguration.identityId) ? cmekConfiguration.identityId : '' -// kvKeyUri: !empty(cmekConfiguration.kvUrl) && !empty(cmekConfiguration.keyName) ? '${cmekConfiguration.kvUrl}/keys/${cmekConfiguration.keyName}' : '' - -// // Persist Secrets to Vault -// keyVaultName: keyvault.outputs.name -// databaseEndpointSecretName: 'graph-db-endpoint' -// databasePrimaryKeySecretName: 'graph-db-primary-key' -// databaseConnectionStringSecretName: 'graph-db-connection' - - -// roleAssignments: [ -// { -// roleDefinitionIdOrName: 'Contributor' -// principals: [ -// { -// id: applicationClientPrincipalOid -// } -// ] -// principalType: 'ServicePrincipal' -// } -// ] -// } -// } - - -/* - _______. ______ .______ __ .______ .___________. _______. - / | / || _ \ | | | _ \ | | / | - | (----`| ,----'| |_) | | | | |_) | `---| |----` | (----` - \ \ | | | / | | | ___/ | | \ \ -.----) | | `----.| |\ \----.| | | | | | .----) | -|_______/ \______|| _| `._____||__| | _| |__| |_______/ -*/ - - -var directoryUploads = [ - { - directory: 'software' - } - { - directory: 'charts' - } - { - directory: 'stamp' - } -] - -@batchSize(1) -module gitOpsUpload './software-upload/main.bicep' = [for item in directoryUploads: { - name: '${bladeConfig.sectionName}-storage-${item.directory}-upload' - params: { - newStorageAccount: true - location: location - storageAccountName: storage.outputs.name - identityName: userAssignedIdentity.name - - directoryName: item.directory - } - dependsOn: [ - storage - ] -}] - -module manifestDagShareUpload './script-share-upload/main.bicep' = { - name: '${bladeConfig.sectionName}-storage-dag-upload-manifest' - params: { - newStorageAccount: true - location: location - storageAccountName: storage.outputs.name - identityName: userAssignedIdentity.name - - shareName: 'airflow-dags' - filename: 'src/osdu_dags' - compress: true - fileurl: 'https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-dags/-/archive/master/ingestion-dags-master.tar.gz' - } - dependsOn: [ - storage - ] -} - -module csvDagShareUpload './script-share-csvdag/main.bicep' = { - name: '${bladeConfig.sectionName}-storage-dag-upload-csv' - params: { - newStorageAccount: true - location: location - storageAccountName: storage.outputs.name - identityName: userAssignedIdentity.name - - shareName: 'airflow-dags' - filename: 'airflowdags' - fileurl: 'https://community.opengroup.org/osdu/platform/data-flow/ingestion/csv-parser/csv-parser/-/archive/master/csv-parser-master.tar.gz' - keyVaultUrl: keyvault.outputs.uri - insightsKey: insightsKey - clientId: applicationClientId - clientSecret: applicationClientSecret - } - dependsOn: [ - storage - ] -} - - -// =============== // -// Outputs // -// =============== // - -output keyvaultName string = keyvault.outputs.name -output keyvaultUri string = keyvault.outputs.uri -output storageAccountName string = storage.outputs.name -output storageAccountResourceId string = storage.outputs.resourceId - - -// =============== // -// Definitions // -// =============== // - -type bladeSettings = { - @description('The name of the section name') - sectionName: string - @description('The display name of the section') - displayName: string -} diff --git a/bicep/modules/blade_partition.bicep b/bicep/modules/blade_partition.bicep index 1f105725..ddb65dd4 100644 --- a/bicep/modules/blade_partition.bicep +++ b/bicep/modules/blade_partition.bicep @@ -504,7 +504,7 @@ module storage 'storage-account/main.bicep' = [for (partition, index) in partiti }) } - + // Apply Roles roleAssignments: [ { roleDefinitionIdOrName: 'Storage Blob Data Contributor' @@ -516,9 +516,7 @@ module storage 'storage-account/main.bicep' = [for (partition, index) in partiti // Apply Security allowBlobPublicAccess: enableBlobPublicAccess publicNetworkAccess: 'Enabled' - - // TODO: Deployment Scripts don't support this yet. - // allowSharedKeyAccess: true + allowSharedKeyAccess: true // https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/deployment-script-template?tabs=CLI#debug-deployment-scripts networkAcls: { bypass: 'AzureServices' @@ -547,47 +545,6 @@ module storage 'storage-account/main.bicep' = [for (partition, index) in partiti }] - -// module partitionStorage './storage-account-orig/main.bicep' = [for (partition, index) in partitions: { -// name: '${bladeConfig.sectionName}-azure-storage-${index}' -// params: { -// #disable-next-line BCP335 BCP332 -// name: '${replace('data${index}${substring(uniqueString(partition.name), 0, 6)}', '-', '')}${uniqueString(resourceGroup().id, 'data${index}${substring(uniqueString(partition.name), 0, 6)}')}' -// location: location - -// // Assign Tags - -// tags: union( -// tags, -// { -// layer: bladeConfig.displayName -// partition: partition.name -// purpose: 'data' -// } -// ) - -// // Hook up Diagnostics -// diagnosticWorkspaceId: workspaceResourceId -// diagnosticLogsRetentionInDays: 0 - -// // Apply Security -// allowBlobPublicAccess: enableBlobPublicAccess - -// // Configure Service -// sku: partitionLayerConfig.storage.sku -// containers: concat(partitionLayerConfig.storage.containers, [partition.name]) - -// // Hookup Customer Managed Encryption Key -// cmekConfiguration: cmekConfiguration - -// // Persist Secrets to Vault -// keyVaultName: kvName -// storageAccountSecretName: '${partition.name}-${partitionLayerConfig.secrets.storageAccountName}' -// storageAccountKeySecretName: '${partition.name}-${partitionLayerConfig.secrets.storageAccountKey}' -// storageAccountBlobEndpointSecretName: '${partition.name}-${partitionLayerConfig.secrets.storageAccountBlob}' -// } -// }] - module partitionDb './cosmos-db/main.bicep' = [for (partition, index) in partitions: { name: '${bladeConfig.sectionName}-cosmos-db-${index}' params: { @@ -609,6 +566,15 @@ module partitionDb './cosmos-db/main.bicep' = [for (partition, index) in partiti diagnosticWorkspaceId: workspaceResourceId diagnosticLogsRetentionInDays: 0 + networkRestrictions: { + publicNetworkAccess: 'Enabled' + networkAclBypass: 'AzureServices' + ipRules: [ + '${natClusterIP}' + ] + virtualNetworkRules: [] + } + // Set isSystemPartition based on index isSystemPartition: index == 0 ? true : false @@ -702,18 +668,32 @@ module partitonNamespace 'br/public:avm/res/service-bus/namespace:0.9.1' = [for }] -// Deployment Scripts are not enabled yet for Private Link -// https://github.com/Azure/bicep/issues/6540 -module blobUpload './script-blob-upload/main.bicep' = [for (partition, index) in partitions: { +module blobUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for (partition, index) in partitions: { name: '${bladeConfig.sectionName}-storage-blob-upload-${index}' params: { - storageAccountName: storage[index].outputs.name + name: 'script-${storage[index].outputs.name}-Legal_COO' location: location + cleanupPreference: 'Always' + retentionInterval: 'PT1H' + timeout: 'PT30M' + runOnce: true + + managedIdentities: { + userAssignedResourcesIds: [ + stampIdentity.id + ] + } - useExistingManagedIdentity: true - managedIdentityName: managedIdentityName - existingManagedIdentitySubId: subscription().subscriptionId - existingManagedIdentityResourceGroupName:resourceGroup().name + kind: 'AzureCLI' + azCliVersion: '2.63.0' + + environmentVariables: [ + { name: 'CONTENT', value: loadTextContent('./deploy-scripts/Legal_COO.json') } + { name: 'FILE_NAME', value: 'Legal_COO.json' } + { name: 'CONTAINER', value: 'legal-service-azure-configuration' } + { name: 'AZURE_STORAGE_ACCOUNT', value: storage[index].outputs.name } + ] + scriptContent: loadTextContent('./deploy-scripts/blob_upload.sh') } }] diff --git a/bicep/modules/cosmos-db/main.bicep b/bicep/modules/cosmos-db/main.bicep index be42a7e2..f4de069e 100644 --- a/bicep/modules/cosmos-db/main.bicep +++ b/bicep/modules/cosmos-db/main.bicep @@ -202,6 +202,26 @@ param kvKeyUri string = '' @description('Optional. Indicates if the module is used in a cross tenant scenario. If true, a resourceId must be provided in the role assignment\'s principal object.') param crossTenant bool = false +@description('Optional. The network configuration of this module. Defaults to `{ ipRules: [], virtualNetworkRules: [], publicNetworkAccess: \'Disabled\' }`.') +param networkRestrictions networkRestrictionsType = { + ipRules: [] + virtualNetworkRules: [] + publicNetworkAccess: 'Disabled' +} + +var ipRules = [ + for i in (networkRestrictions.?ipRules ?? []): { + ipAddressOrRange: i + } +] + +var virtualNetworkRules = [ + for vnet in (networkRestrictions.?virtualNetworkRules ?? []): { + id: vnet.subnetResourceId + ignoreMissingVnetServiceEndpoint: false + } +] + var name = '${replace(resourceName, '-', '')}${uniqueString(resourceGroup().id, resourceName)}' @@ -286,6 +306,13 @@ var databaseAccount_properties = union({ } ] : databaseAccount_locations + + ipRules: ipRules + virtualNetworkRules: virtualNetworkRules + networkAclBypass: networkRestrictions.?networkAclBypass ?? 'AzureServices' + publicNetworkAccess: networkRestrictions.?publicNetworkAccess ?? 'Enabled' + isVirtualNetworkFilterEnabled: !empty(ipRules) || !empty(virtualNetworkRules) + capabilities: capabilities backupPolicy: backupPolicy } : {}), (!empty(sqlDatabases) ? { @@ -557,3 +584,21 @@ module secretDatabaseConnectionString '.bicep/keyvault_secrets.bicep' = if (!em value: databaseAccount.listConnectionStrings().connectionStrings[0].connectionString } } + + +type networkRestrictionsType = { + @description('Required. A single IPv4 address or a single IPv4 address range in CIDR format. Provided IPs must be well-formatted and cannot be contained in one of the following ranges: 10.0.0.0/8, 100.64.0.0/10, 172.16.0.0/12, 192.168.0.0/16, since these are not enforceable by the IP address filter. Example of valid inputs: "23.40.210.245" or "23.40.210.0/8".') + ipRules: string[] + + @description('Optional. Default to AzureServices. Specifies the network ACL bypass for Azure services.') + networkAclBypass: ('AzureServices' | 'None')? + + @description('Optional. Default to Enabled. Whether requests from Public Network are allowed.') + publicNetworkAccess: ('Enabled' | 'Disabled')? + + @description('Required. List of Virtual Network ACL rules configured for the Cosmos DB account..') + virtualNetworkRules: { + @description('Required. Resource ID of a subnet.') + subnetResourceId: string + }[] +} diff --git a/bicep/modules/script-blob-upload/Legal_COO.json b/bicep/modules/deploy-scripts/Legal_COO.json similarity index 100% rename from bicep/modules/script-blob-upload/Legal_COO.json rename to bicep/modules/deploy-scripts/Legal_COO.json diff --git a/bicep/modules/script-blob-upload/script.sh b/bicep/modules/deploy-scripts/blob_upload.sh similarity index 74% rename from bicep/modules/script-blob-upload/script.sh rename to bicep/modules/deploy-scripts/blob_upload.sh index d259c604..e9dfb462 100644 --- a/bicep/modules/script-blob-upload/script.sh +++ b/bicep/modules/deploy-scripts/blob_upload.sh @@ -1,14 +1,12 @@ #!/bin/bash set -e -echo "Waiting on Identity RBAC replication ($initialDelay)" -sleep $initialDelay - # Installing curl apk add --no-cache curl echo "$CONTENT" > ${FILE_NAME} # Upload the blob, overwriting if it exists -az storage blob upload -f ${FILE_NAME} -c ${CONTAINER} -n ${FILE_NAME} --overwrite +az login --identity +az storage blob upload -f ${FILE_NAME} -c ${CONTAINER} -n ${FILE_NAME} --overwrite --auth-mode login echo "Blob ${CONTAINER} uploaded to container ${CONTAINER}, overwriting if it existed." \ No newline at end of file diff --git a/bicep/modules/deploy-scripts/software-upload.sh b/bicep/modules/deploy-scripts/software-upload.sh new file mode 100644 index 00000000..d0e1b72d --- /dev/null +++ b/bicep/modules/deploy-scripts/software-upload.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -e + +# Installing required packages +apk add --no-cache curl zip unzip + +# Download the file using curl +echo "Downloading file from ${URL}" +curl -L -o repo.zip "${URL}" + +# Check if the download was successful +if [ $? -ne 0 ]; then + echo "Failed to download the file from ${URL}" + exit 1 +fi + +# Create a directory for extracted files +mkdir -p extracted_files + +# Unzip the file +echo "Extracting contents..." +unzip -q repo.zip -d extracted_files + +# Find and replace 'kind: GitRepository' with 'kind: Bucket' in all files +find extracted_files -type f -path "*/${UPLOAD_DIR}/*" -exec sed -i ' + /sourceRef:/{ + N;N;N + s/sourceRef:\n[[:space:]]*kind: GitRepository\n[[:space:]]*name: flux-system\n[[:space:]]*namespace: flux-system/sourceRef:\n kind: Bucket\n name: flux-system\n namespace: flux-system/g + }' {} + + +# Find the software directory +software_dir=$(find extracted_files -type d -name "${UPLOAD_DIR}" -exec dirname {} \;) + +if [ -z "$software_dir" ]; then + echo "Error: '${UPLOAD_DIR}' directory not found in the extracted contents." + exit 1 +fi + +# Upload the contents of the software directory +echo "Uploading files from ${software_dir} to blob container ${CONTAINER}" +az storage blob upload-batch --destination ${CONTAINER} --source "${software_dir}" --pattern "${UPLOAD_DIR}/**" --overwrite true --auth-mode login +echo "Files from software directory uploaded to blob container ${CONTAINER}." + +# Clean up +rm -rf extracted_files repo.zip diff --git a/bicep/modules/script-blob-upload/main.bicep b/bicep/modules/script-blob-upload/main.bicep deleted file mode 100644 index ea7d2507..00000000 --- a/bicep/modules/script-blob-upload/main.bicep +++ /dev/null @@ -1,94 +0,0 @@ -metadata name = 'Blob Upload' -metadata description = 'This module uploads a file to a blob storage account' -metadata owner = 'azure-global-energy' - -@description('Desired name of the storage account') -param storageAccountName string = uniqueString(resourceGroup().id, deployment().name, 'blob') - -@description('Name of the blob container') -param containerName string = 'legal-service-azure-configuration' - -@description('Name of the blob as it is stored in the blob container') -param filename string = 'Legal_COO.json' - -@description('The location of the Storage Account and where to deploy the module resources to') -param location string = resourceGroup().location - -@description('How the deployment script should be forced to execute') -param forceUpdateTag string = utcNow() - -@description('Azure RoleId that are required for the DeploymentScript resource to upload blobs') -param rbacRoleNeeded string = '' //Storage Blob Contributor is needed to upload secrets into Storage Account - -@description('Does the Managed Identity already exists, or should be created') -param useExistingManagedIdentity bool = false - -@description('Name of the Managed Identity resource') -param managedIdentityName string = 'id-storage-blob-${location}' - -@description('For an existing Managed Identity, the Subscription Id it is located in') -param existingManagedIdentitySubId string = subscription().subscriptionId - -@description('For an existing Managed Identity, the Resource Group it is located in') -param existingManagedIdentityResourceGroupName string = resourceGroup().name - -@description('A delay before the script import operation starts. Primarily to allow Azure AAD Role Assignments to propagate') -param initialScriptDelay string = '30s' - -@allowed([ 'OnSuccess', 'OnExpiration', 'Always' ]) -@description('When the script resource is cleaned up') -param cleanupPreference string = 'OnSuccess' - - -resource storageAccount 'Microsoft.Storage/storageAccounts@2023-04-01' existing = { - name: storageAccountName -} - -resource newDepScriptId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = if (!useExistingManagedIdentity) { - name: managedIdentityName - location: location -} - -resource existingDepScriptId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = if (useExistingManagedIdentity) { - name: managedIdentityName - scope: resourceGroup(existingManagedIdentitySubId, existingManagedIdentityResourceGroupName) -} - -resource rbac 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (!empty(rbacRoleNeeded)) { - name: guid(storageAccount.id, rbacRoleNeeded, useExistingManagedIdentity ? existingDepScriptId.id : newDepScriptId.id) - scope: storageAccount - properties: { - roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', rbacRoleNeeded) - principalId: useExistingManagedIdentity ? existingDepScriptId.properties.principalId : newDepScriptId.properties.principalId - principalType: 'ServicePrincipal' - } -} - -resource uploadFile 'Microsoft.Resources/deploymentScripts@2023-08-01' = { - name: 'script-${storageAccount.name}-${replace(replace(filename, ':', ''), '/', '-')}' - location: location - identity: { - type: 'UserAssigned' - userAssignedIdentities: { '${useExistingManagedIdentity ? existingDepScriptId.id : newDepScriptId.id}': {} } - } - kind: 'AzureCLI' - dependsOn: [ rbac ] - properties: { - forceUpdateTag: forceUpdateTag - azCliVersion: '2.63.0' - timeout: 'PT30M' - retentionInterval: 'PT1H' - environmentVariables: [ - { name: 'AZURE_STORAGE_ACCOUNT', value: storageAccount.name } - { name: 'AZURE_STORAGE_KEY', value: storageAccount.listKeys().keys[0].value } - { name: 'CONTENT', value: loadTextContent('./Legal_COO.json') } - { name: 'FILE_NAME', value: filename } - { name: 'CONTAINER', value: containerName } - { name: 'initialDelay', value: initialScriptDelay } - ] - scriptContent: loadTextContent('script.sh') - // scriptContent: 'echo "$CONTENT" > ${filename} && az storage blob upload -f ${filename} -c ${containerName} -n ${filename}' - cleanupPreference: cleanupPreference - } -} - diff --git a/bicep/modules/script-kv-certificate/README.md b/bicep/modules/script-kv-certificate/README.md deleted file mode 100644 index ee6f6268..00000000 --- a/bicep/modules/script-kv-certificate/README.md +++ /dev/null @@ -1,147 +0,0 @@ -# Key Vault Certificate - -This module creates a Key Vault Certificate and stores it in an Azure Key Vault - -## Details - -{{Add detailed information about the module}} - -## Parameters - -| Name | Type | Required | Description | -| :----------------------------------------- | :------------: | :------: | :------------------------------------------------------------------------------------------------------------ | -| `kvName` | `string` | Yes | The name of the Azure Key Vault | -| `location` | `string` | Yes | The location to deploy the resources to | -| `forceUpdateTag` | `string` | No | How the deployment script should be forced to execute | -| `rbacRolesNeededOnKV` | `string` | No | The RoleDefinitionId required for the DeploymentScript resource to interact with KeyVault | -| `useExistingManagedIdentity` | `bool` | No | Does the Managed Identity already exists, or should be created | -| `managedIdentityName` | `string` | No | Name of the Managed Identity resource | -| `existingManagedIdentitySubId` | `string` | No | For an existing Managed Identity, the Subscription Id it is located in | -| `existingManagedIdentityResourceGroupName` | `string` | No | For an existing Managed Identity, the Resource Group it is located in | -| `certificateNames` | `array` | Yes | The names of the certificate to create. Use when creating many certificates. | -| `certificateCommonNames` | `array` | No | The common names of the certificate to create. Use when creating many certificates. | -| `initialScriptDelay` | `string` | No | A delay before the script import operation starts. Primarily to allow Azure AAD Role Assignments to propagate | -| `cleanupPreference` | `string` | No | When the script resource is cleaned up | -| `issuerName` | `string` | No | Self, or user defined {IssuerName} for certificate signing | -| `issuerProvider` | `string` | No | Certificate Issuer Provider, DigiCert, GlobalSign, or internal options may be used. | -| `disabled` | `bool` | No | Create certificate in disabled state. Default: false | -| `accountId` | `string` | No | Account ID of Certificate Issuer Account | -| `issuerPassword` | `securestring` | No | Password of Certificate Issuer Account | -| `organizationId` | `string` | No | Organization ID of Certificate Issuer Account | -| `isCrossTenant` | `bool` | No | Override this parameter if using this in cross tenant scenarios | -| `reuseKey` | `bool` | No | The default policy might cause errors about CSR being used before, so set this to false if that happens | -| `validity` | `int` | No | Optional. Override default validityInMonths 12 value | -| `performRoleAssignment` | `bool` | No | Set to false to disable role assignments within this module. Default: true | - -## Outputs - -| Name | Type | Description | -| :-------------------------------- | :-----: | :------------------------------------------------- | -| `certificateNames` | `array` | Certificate names | -| `certificateSecretIds` | `array` | KeyVault secret ids to the created version | -| `certificateSecretIdUnversioneds` | `array` | KeyVault secret ids which uses the unversioned uri | -| `certificateThumbpints` | `array` | Certificate Thumbprints | -| `certificateThumbprintHexs` | `array` | Certificate Thumbprints (in hex) | - -## Examples - -### Single KeyVault Certificate - -Creates a single self-signed certificate in Azure KeyVault. - -```bicep -param location string = resourceGroup().location -param akvName string = 'yourAzureKeyVault' -param certificateName string = 'myapp' - -module kvCert 'br/public:deployment-scripts/create-kv-certificate:3.4' = { - name: 'akvCertSingle' - params: { - akvName: akvName - location: location - certificateName: certificateName - } -} -output SecretId string = akvCertSingle.outputs.certificateSecretId -output Thumbprint string = akvCertSingle.outputs.certificateThumbprintHex - -``` - -### Single KeyVault Certificate with fqdn common name - -Creates a single self-signed certificate in Azure KeyVault using a specific certificate common name. - -```bicep -param location string = resourceGroup().location -param akvName string = 'yourAzureKeyVault' -param certificateName string = 'myapp' -param certificateCommonName string = '${certificateName}.mydomain.local' - -module kvCert 'br/public:deployment-scripts/create-kv-certificate:3.4' = { - name: 'akvCertSingle' - params: { - akvName: akvName - location: location - certificateNames: [certificateName] - certificateCommonNames: [certificateCommonName] - } -} -output SecretId string = akvCertSingle.outputs.certificateSecretIds[0] -output Thumbprint string = akvCertSingle.outputs.certificateThumbprintHexs[0] - -``` - -### Multiple KeyVault Certificates - -Create multiple self-signed certificates in Azure KeyVault - -```bicep -param location string = resourceGroup().location -param akvName string = 'yourAzureKeyVault' -param certificateNames array = [ - 'myapp' - 'myotherapp' -] - -module kvCert 'br/public:deployment-scripts/create-kv-certificate:3.1.1' = { - name: 'akvCert-${certificateName}' - params: { - akvName: akvName - location: location - certificateNames: certificateNames - } -} - -@description('Array of info from each Certificate') -output createdCertificates array = [for (certificateName, i) in certificateNames: { - certificateName: kvCert.outputs.certificateNames[i] - certificateSecretId: kvCert.outputs.certificateSecretIds[i] - certificateThumbprint: kvCert.outputs.certificateThumbprintHexs[i] -}] -``` - -### Create Signed Certificate - -Using `DigiCert` or `GlobalSign` first requires account setup described [here](https://learn.microsoft.com/en-us/azure/key-vault/certificates/how-to-integrate-certificate-authority) - -```bicep -param accountId -@secure -param issuerPassword -param organizationId - -module signedCert 'br/public:deployment-scripts/create-kv-certificate:3.1.1' = { - name: 'akvCert-${certificateName}' - params: { - akvName: akvName - location: location - certificateName: [certificateName] - certificateCommonName: ['customdomain.com'] - issuerName: 'MyCert' - issuerProvider: 'DigiCert' - accountId: accountId - issuerPassword: issuerPassword - organizationId: organizationId - } -}] -``` \ No newline at end of file diff --git a/bicep/modules/script-kv-certificate/main.bicep b/bicep/modules/script-kv-certificate/main.bicep deleted file mode 100644 index c9b97a89..00000000 --- a/bicep/modules/script-kv-certificate/main.bicep +++ /dev/null @@ -1,170 +0,0 @@ -metadata name = 'Key Vault Certificate' -metadata description = 'This module creates a Key Vault Certificate and stores it in an Azure Key Vault' -metadata owner = 'azure-global-energy' - -@description('The name of the Azure Key Vault') -param kvName string - -@description('The location to deploy the resources to') -param location string - -@description('How the deployment script should be forced to execute') -param forceUpdateTag string = utcNow() - -@description('The RoleDefinitionId required for the DeploymentScript resource to interact with KeyVault') -param rbacRolesNeededOnKV string = 'a4417e6f-fecd-4de8-b567-7b0420556985' //KeyVault Certificate Officer - -@description('Does the Managed Identity already exists, or should be created') -param useExistingManagedIdentity bool = false - -@description('Name of the Managed Identity resource') -param managedIdentityName string = 'id-KeyVaultCertificateCreator-${location}' - -@description('For an existing Managed Identity, the Subscription Id it is located in') -param existingManagedIdentitySubId string = subscription().subscriptionId - -@description('For an existing Managed Identity, the Resource Group it is located in') -param existingManagedIdentityResourceGroupName string = resourceGroup().name - -@description('The names of the certificate to create. Use when creating many certificates.') -param certificateNames array - -@description('The common names of the certificate to create. Use when creating many certificates.') -param certificateCommonNames array = certificateNames - -@description('A delay before the script import operation starts. Primarily to allow Azure AAD Role Assignments to propagate') -param initialScriptDelay string = '0' - -@allowed([ - 'OnSuccess' - 'OnExpiration' - 'Always' -]) -@description('When the script resource is cleaned up') -param cleanupPreference string = 'OnSuccess' - -@description('Self, or user defined {IssuerName} for certificate signing') -param issuerName string = 'Self' - -@description('Certificate Issuer Provider, DigiCert, GlobalSign, or internal options may be used.') -param issuerProvider string = '' - -@description('Create certificate in disabled state. Default: false') -param disabled bool = false - -@description('Account ID of Certificate Issuer Account') -param accountId string = '' - -@description('Password of Certificate Issuer Account') -@secure() -param issuerPassword string = '' - -@description('Organization ID of Certificate Issuer Account') -param organizationId string = '' - -@description('Override this parameter if using this in cross tenant scenarios') -param isCrossTenant bool = false - -@description('The default policy might cause errors about CSR being used before, so set this to false if that happens') -param reuseKey bool = true - -@minValue(1) -@maxValue(1200) -@description('Optional. Override default validityInMonths 12 value') -param validity int = 12 - -@description('Set to false to disable role assignments within this module. Default: true') -param performRoleAssignment bool = true - -resource akv 'Microsoft.KeyVault/vaults@2022-07-01' existing = { - name: kvName -} - -@description('A new managed identity that will be created in this Resource Group, this is the default option') -resource newDepScriptId 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = if (!useExistingManagedIdentity) { - name: managedIdentityName - location: location -} - -@description('An existing managed identity that could exist in another sub/rg') -resource existingDepScriptId 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' existing = if (useExistingManagedIdentity) { - name: managedIdentityName - scope: resourceGroup(existingManagedIdentitySubId, existingManagedIdentityResourceGroupName) -} - -var delegatedManagedIdentityResourceId = useExistingManagedIdentity ? existingDepScriptId.id : newDepScriptId.id - -resource rbacKv 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (performRoleAssignment) { - name: guid(akv.id, rbacRolesNeededOnKV, managedIdentityName, string(useExistingManagedIdentity)) - scope: akv - properties: { - roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', rbacRolesNeededOnKV) - principalId: useExistingManagedIdentity ? existingDepScriptId.properties.principalId : newDepScriptId.properties.principalId - principalType: 'ServicePrincipal' - delegatedManagedIdentityResourceId: isCrossTenant ? delegatedManagedIdentityResourceId : null - } -} - -resource createImportCerts 'Microsoft.Resources/deploymentScripts@2023-08-01' = [for (certificateName, index) in certificateNames: { - name: 'script-${akv.name}-${replace(replace(certificateName, ':', ''), '/', '-')}' - location: location - identity: { - type: 'UserAssigned' - userAssignedIdentities: { - '${useExistingManagedIdentity ? existingDepScriptId.id : newDepScriptId.id}': {} - } - } - kind: 'AzureCLI' - dependsOn: [ - rbacKv - ] - properties: { - forceUpdateTag: forceUpdateTag - azCliVersion: '2.63.0' - timeout: 'PT30M' - retentionInterval: 'PT1H' - environmentVariables: [ - { name: 'akvName', value: kvName } - { name: 'certName', value: certificateName } - { name: 'certCommonName', value: certificateCommonNames[index] } - { name: 'initialDelay', value: initialScriptDelay } - { name: 'issuerName', value: issuerName } - { name: 'issuerProvider', value: issuerProvider } - { name: 'disabled', value: toLower(string(disabled)) } - { name: 'retryMax', value: '10' } - { name: 'retrySleep', value: '5s' } - { name: 'accountId', value: accountId } - { name: 'issuerPassword', secureValue: issuerPassword } - { name: 'organizationId', value: organizationId } - { name: 'reuseKey', value: toLower(string(reuseKey)) } - { name: 'validity', value: string(validity) } - ] - scriptContent: loadTextContent('script.sh') - cleanupPreference: cleanupPreference - } -}] - -@description('Certificate names') -output certificateNames array = [for (certificateName, index) in certificateNames: [ - createImportCerts[index].properties.outputs.name -]] - -@description('KeyVault secret ids to the created version') -output certificateSecretIds array = [for (certificateName, index) in certificateNames: [ - createImportCerts[index].properties.outputs.certSecretId.versioned -]] - -@description('KeyVault secret ids which uses the unversioned uri') -output certificateSecretIdUnversioneds array = [for (certificateName, index) in certificateNames: [ - createImportCerts[index].properties.outputs.certSecretId.unversioned -]] - -@description('Certificate Thumbprints') -output certificateThumbpints array = [for (certificateName, index) in certificateNames: [ - createImportCerts[index].properties.outputs.thumbprint -]] - -@description('Certificate Thumbprints (in hex)') -output certificateThumbprintHexs array = [for (certificateName, index) in certificateNames: [ - createImportCerts[index].properties.outputs.thumbprintHex -]] diff --git a/bicep/modules/script-kv-certificate/script.sh b/bicep/modules/script-kv-certificate/script.sh deleted file mode 100644 index 93838ad5..00000000 --- a/bicep/modules/script-kv-certificate/script.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash -set -e -initialDelay="${initialDelay:-5}" -retryMax="${retryMax:-5}" -certName="${certName:-default-cert}" -certCommonName="${certCommonName:-default}" -validity="${validity:-12}" -akvName="${akvName:-keyvault}" -issuerName="${issuerName:-}" -reuseKey="${reuseKey:-true}" -retrySleep="${retrySleep:-5}" - -echo "Waiting on Identity RBAC replication (\"$initialDelay\")" -sleep "$initialDelay" - -#Retry loop to catch errors (usually RBAC delays) -retryLoopCount=0 -until [ "$retryLoopCount" -ge "$retryMax" ] -do - echo "Creating AKV Cert $certName with CN $certCommonName (attempt $retryLoopCount)..." - - if [ -z "$issuerName" ] || [ -z "$issuerProvider" ]; then - policy=$(az keyvault certificate get-default-policy \ - | sed -e s/\"validityInMonths\":\ 12/\"validityInMonths\":\ "${validity}"/g \ - | sed -e s/CN=CLIGetDefaultPolicy/CN="${certCommonName}"/g ) - else - if [ "$issuerProvider" == "DigiCert" ] || [ "$issuerProvider" == "GlobalCert" ]; then - az keyvault certificate issuer create \ - --vault-name "$akvName" \ - --issuer-name "$issuerName" \ - --provider-name "$issuerProvider" \ - --account-id "$accountId" \ - --password "$issuerPassword" \ - --organizatiion-id "$organizationId" - else - az keyvault certificate issuer create \ - --vault-name "$akvName" \ - --issuer-name "$issuerName" \ - --provider-name "$issuerProvider" - fi - policy=$(az keyvault certificate get-default-policy \ - | sed -e s/\"validityInMonths\":\ 12/\"validityInMonths\":\ "${validity}"/g \ - | sed -e s/CN=CLIGetDefaultPolicy/CN="${certCommonName}"/g \ - | sed -e s/\"name\":\ \"Self\"/\"name\":\ \""${issuerName}"\"/g \ - | sed -e s/\"reuseKey\":\ true/\"reuseKey\":\ "${reuseKey}"/g ) - fi - az keyvault certificate create \ - --vault-name "$akvName" \ - -n "$certName" \ - -p "$policy" \ - --disabled "$disabled" \ - && break - - sleep "$retrySleep" - retryLoopCount=$((retryLoopCount+1)) -done - -echo "Getting Certificate $certName"; -retryLoopCount=0 -createdCert=$(az keyvault certificate show -n "$certName" --vault-name "$akvName" -o json) -while [ -z "$(echo "$createdCert" | jq -r '.x509ThumbprintHex')" ] && [ $retryLoopCount -lt "$retryMax" ] -do - echo "Waiting for cert creation (attempt $retryLoopCount)..." - sleep $retrySleep - createdCert=$(az keyvault certificate show -n $certName --vault-name $akvName -o json) - retryLoopCount=$((retryLoopCount+1)) -done - -unversionedSecretId=$(echo $createdCert | jq -r ".sid" | cut -d'/' -f-5) # remove the version from the url; -jsonOutputString=$(echo $createdCert | jq --arg usid $unversionedSecretId '{name: .name ,certSecretId: {versioned: .sid, unversioned: $usid }, thumbprint: .x509Thumbprint, thumbprintHex: .x509ThumbprintHex}') -echo $jsonOutputString > $AZ_SCRIPTS_OUTPUT_PATH \ No newline at end of file diff --git a/bicep/modules/script-kv-certificate/test/main.test.bicep b/bicep/modules/script-kv-certificate/test/main.test.bicep deleted file mode 100644 index 5bd3e973..00000000 --- a/bicep/modules/script-kv-certificate/test/main.test.bicep +++ /dev/null @@ -1,104 +0,0 @@ -targetScope = 'resourceGroup' - -@minLength(3) -@maxLength(10) -@description('Used to name all resources') -param resourceName string - -@description('Registry Location.') -param location string = resourceGroup().location - -//Prerequisites -module identity '../../user-managed-identity/main.bicep' = { - name: 'user-managed-identity' - params: { - resourceName: resourceName - location: location - } -} - -module kv '../../azure-keyvault/main.bicep' = { - name: 'azure_keyvault' - params: { - resourceName: resourceName - location: location - secretsObject: { secrets: [] } - - // Add Role Assignment - roleAssignments: [ - { - roleDefinitionIdOrName: 'Key Vault Administrator' - principals: [ - { - id: identity.outputs.principalId - } - ] - principalType: 'ServicePrincipal' - } - ] - } -} - -//Test 1. Just a single certificate -module kvCertSingle '../main.bicep' = { - name: 'kvCertSingle' - params: { - kvName: kv.outputs.name - location: location - - useExistingManagedIdentity: true - managedIdentityName: identity.outputs.name - existingManagedIdentitySubId: subscription().subscriptionId - existingManagedIdentityResourceGroupName:resourceGroup().name - - certificateNames: [ 'mysingleapp' ] - certificateCommonNames: [ 'mysingleapp.mydomain.local' ] - validity: 11 - disabled: true - } -} -output singleSecretId string = kvCertSingle.outputs.certificateSecretIds[0][0] -output singleThumbprint string = kvCertSingle.outputs.certificateThumbprintHexs[0][0] - -//Test 2. Array of certificates -var certificateNames = [ - 'myapp' - 'myotherapp' -] - -module kvCertMultiple '../main.bicep' = { - name: 'kvCertMultiple-${uniqueString(kv.name)}' - params: { - kvName: kv.name - location: location - - useExistingManagedIdentity: true - managedIdentityName: identity.outputs.name - existingManagedIdentitySubId: subscription().subscriptionId - existingManagedIdentityResourceGroupName:resourceGroup().name - - certificateNames: certificateNames - initialScriptDelay: '0' - validity: 24 - } -} - -// Test 3. Test a signed cert -// module akvCertSigned '../main.bicep' = { -// name: 'akvCertSigned' -// params: { -// akvName: akv.name -// location: location -// certificateName: 'mysignedcert' -// certificateCommonName: 'sample-cert.gaming.azure.com' -// issuerName: 'Signed' -// issuerProvider: 'OneCertV2-PublicCA' -// } -// } - -@description('Array of info from each Certificate') -output createdCertificates array = [for (certificateName, i) in certificateNames: { - certificateName: certificateName - certificateSecretId: kvCertMultiple.outputs.certificateSecretIds[i] - certificateThumbprint: kvCertMultiple.outputs.certificateThumbprintHexs[i] -}] diff --git a/bicep/modules/script-kv-certificate/test/parameters.json b/bicep/modules/script-kv-certificate/test/parameters.json deleted file mode 100644 index 4d4c4d1d..00000000 --- a/bicep/modules/script-kv-certificate/test/parameters.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "template": "../main.bicep" - }, - "parameters": { - "location": { - "value": "southcentralus" - }, - "resourceName": { - "value": "adme" - } - } -} \ No newline at end of file diff --git a/bicep/modules/script-kv-certificate/version.json b/bicep/modules/script-kv-certificate/version.json deleted file mode 100644 index e215964e..00000000 --- a/bicep/modules/script-kv-certificate/version.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "$schema": "https://aka.ms/bicep-registry-module-version-file-schema#", - "version": "3.4", - "pathFilters": [ - "./main.json" - ] -} \ No newline at end of file diff --git a/bicep/modules/script-share-csvdag/main.bicep b/bicep/modules/script-share-csvdag/main.bicep index d05b7e8f..5975709e 100644 --- a/bicep/modules/script-share-csvdag/main.bicep +++ b/bicep/modules/script-share-csvdag/main.bicep @@ -130,7 +130,7 @@ module deploymentScript 'br/public:avm/res/resources/deployment-script:0.4.0' = environmentVariables: [ { name: 'AZURE_STORAGE_ACCOUNT', value: storageAccount.name } - { name: 'AZURE_STORAGE_KEY', value: storageAccount.listKeys().keys[0].value } + // { name: 'AZURE_STORAGE_KEY', value: storageAccount.listKeys().keys[0].value } { name: 'FILE', value: filename } { name: 'URL', value: fileurl } { name: 'SHARE', value: shareName } diff --git a/bicep/modules/script-share-csvdag/script.sh b/bicep/modules/script-share-csvdag/script.sh index ebab9ba1..62d9e7e0 100644 --- a/bicep/modules/script-share-csvdag/script.sh +++ b/bicep/modules/script-share-csvdag/script.sh @@ -140,5 +140,5 @@ cd "extracted_files/${FILE}" || exit 1 zip -r "${current_dir}/${zip_filename}" . cd - || exit 1 -az storage file upload -s "${SHARE}" --source "${zip_filename}" -onone +az storage file upload -s "${SHARE}" --source "${zip_filename}" --enable-file-backup-request-intent --auth-mode login -onone echo "Zip file ${zip_filename} uploaded to file share ${SHARE}." \ No newline at end of file diff --git a/bicep/modules/script-share-upload/main.bicep b/bicep/modules/script-share-upload/main.bicep index cbc1d3f1..66698ad5 100644 --- a/bicep/modules/script-share-upload/main.bicep +++ b/bicep/modules/script-share-upload/main.bicep @@ -75,7 +75,7 @@ module deploymentScript 'br/public:avm/res/resources/deployment-script:0.4.0' = environmentVariables: [ { name: 'AZURE_STORAGE_ACCOUNT', value: storageAccount.name } - { name: 'AZURE_STORAGE_KEY', value: storageAccount.listKeys().keys[0].value } + // { name: 'AZURE_STORAGE_KEY', value: storageAccount.listKeys().keys[0].value } { name: 'FILE', value: filename } { name: 'URL', value: fileurl } { name: 'SHARE', value: shareName } diff --git a/bicep/modules/script-share-upload/script.sh b/bicep/modules/script-share-upload/script.sh index ea47f1c4..3e0404bf 100644 --- a/bicep/modules/script-share-upload/script.sh +++ b/bicep/modules/script-share-upload/script.sh @@ -43,17 +43,17 @@ if [[ ${URL} == *.tar.gz ]]; then # Navigate back to the original directory cd ${original_dir} # Upload the zip file to the file share - az storage file upload -s ${SHARE} --source ./${zip_filename} -onone + az storage file upload -s ${SHARE} --source ./${zip_filename} --enable-file-backup-request-intent --auth-mode login -onone echo "Zip file ${zip_filename} uploaded to file share ${SHARE}." else # Batch upload the extracted files to the file share using the specified pattern echo "Uploading extracted files to file share ${SHARE} with pattern ${FILE}/**" - az storage file upload-batch -d ${SHARE} --source extracted_files --pattern "${FILE}/**" --no-progress -onone + az storage file upload-batch -d ${SHARE} --source extracted_files --pattern "${FILE}/**" --no-progress --enable-file-backup-request-intent --auth-mode login -onone fi echo "Files from ${url_basename} uploaded to file share ${SHARE}." else # Upload the file to the file share, overwriting if it exists echo "Uploading file ${FILE} to file share ${SHARE}" - az storage file upload -s ${SHARE} --source ./${FILE} -onone + az storage file upload -s ${SHARE} --source ./${FILE} --enable-file-backup-request-intent --auth-mode login -onone echo "File ${FILE} uploaded to file share ${SHARE}, overwriting if it existed." fi \ No newline at end of file diff --git a/bicep/modules/storage-account-orig/.bicep/keyvault_secrets.bicep b/bicep/modules/storage-account-orig/.bicep/keyvault_secrets.bicep deleted file mode 100644 index e457c8e7..00000000 --- a/bicep/modules/storage-account-orig/.bicep/keyvault_secrets.bicep +++ /dev/null @@ -1,31 +0,0 @@ -@description('Conditional. The name of the parent key vault. Required if the template is used in a standalone deployment.') -param keyVaultName string - -@description('Required. The name of the secret.') -param name string - -@description('Required. The value of the secret. NOTE: "value" will never be returned from the service, as APIs using this model are is intended for internal use in ARM deployments. Users should use the data-plane REST service for interaction with vault secrets.') -@secure() -param value string - -resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' existing = { - name: keyVaultName -} - -resource secret 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = { - name: name - parent: keyVault - - properties: { - value: value - } -} - -@description('The name of the secret.') -output name string = secret.name - -@description('The resource ID of the secret.') -output resourceId string = secret.id - -@description('The name of the resource group the secret was created in.') -output resourceGroupName string = resourceGroup().name diff --git a/bicep/modules/storage-account-orig/.bicep/nested_rbac.bicep b/bicep/modules/storage-account-orig/.bicep/nested_rbac.bicep deleted file mode 100644 index 7ee193d6..00000000 --- a/bicep/modules/storage-account-orig/.bicep/nested_rbac.bicep +++ /dev/null @@ -1,70 +0,0 @@ -param description string = '' -param principalType string = '' -param roleDefinitionIdOrName string -param resourceId string - -@sys.description('Optional. Indicates if the module is used in a cross tenant scenario. If true, a resourceId must be provided in the role assignment\'s principal object.') -param crossTenant bool = false - -@sys.description('Required. The IDs of the principals to assign the role to. A resourceId is required when used in a cross tenant scenario (i.e. crossTenant is true)') -param principals array - /* example - [ - { - id: '222222-2222-2222-2222-2222222222' - resourceId: '/subscriptions/111111-1111-1111-1111-1111111111/resourcegroups/rg-osdu-bicep/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-ManagedIdentityName' - } - ] - */ - -var builtInRoleNames = { - Owner: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') - Contributor: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c') - Reader: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'acdd72a7-3385-48ef-bd42-f606fba81ae7') - 'Avere Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4f8fab4f-1852-4a58-a46a-8eaf358af14a') - 'Avere Operator': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'c025889f-8102-4ebf-b32c-fc0c6f0c6bd9') - 'Backup Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '5e467623-bb1f-42f4-a55d-6e525e11384b') - 'Backup Operator': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '00c29273-979b-4161-815c-10b084fb9324') - 'Backup Reader': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'a795c7a0-d4a2-40c1-ae25-d81f01202912') - 'Classic Storage Account Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '') - 'Classic Storage Account Key Operator Service Role': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '') - 'Data Box Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'add466c9-e687-43fc-8d98-dfcf8d720be5') - 'Data Box Reader': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '028f4ed7-e2a9-465e-a8f4-9c0ffdfdc027') - 'Data Lake Analytics Developer': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '47b7735b-770e-4598-a7da-8b91488b4c88') - 'Elastic SAN Owner': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '80dcbedb-47ef-405d-95bd-188a1b4ac406') - 'Elastic SAN Reader': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'af6a70f8-3c9f-4105-acf1-d719e9fca4ca') - 'Elastic SAN Volume Group Owner': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'a8281131-f312-4f34-8d98-ae12be9f0d23') - 'Reader and Data Access': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'c12c1c16-33a1-487b-954d-41c89c60f349') - 'Storage Account Backup Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'e5e2a7ff-d759-4cd2-bb51-3152d37e2eb1') - 'Storage Account Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'e5e2a7ff-d759-4cd2-bb51-3152d37e2eb1') - 'Storage Account Key Operator Service Role': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '81a9662b-bebf-436f-a333-f67b29880f12') - 'Storage Blob Data Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe') - 'Storage Blob Data Owner': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b7e6dc6d-f1e8-4753-8033-0f276bb0955b') - 'Storage Blob Data Reader': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b7e6dc6d-f1e8-4753-8033-0f276bb0955b') - 'Storage Blob Delegator': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'db58b8e5-c6ad-4a2a-8342-4190687cbf4a') - 'Storage File Data SMB Share Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '0c867c2a-1d8c-454a-a3db-ab2ea1bdc8bb') - 'Storage File Data SMB Share Elevated Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'a7264617-510b-434b-a828-9731dc254ea7') - 'Storage File Data SMB Share Reader': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'aba4ae5f-2193-4029-9191-0cb91df5e314') - 'Storage Queue Data Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '974c5e8b-45b9-4653-ba55-5f855dd0fb88') - 'Storage Queue Data Message Processor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '8a0f0c08-91a1-4084-bc3d-661d67233fed') - 'Storage Queue Data Message Sender': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'c6a89b2d-59bc-44d0-9896-0f6e12d7b80a') - 'Storage Queue Data Reader': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '19e7f393-937e-4f77-808e-94535e297925') - 'Storage Table Data Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '0a9a7e1f-b9d0-4cc4-a60d-0319b160aaa3') - 'Storage Table Data Reader': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '76199698-9eea-4c19-bc75-cec21354c6b6') -} - -resource storage 'Microsoft.Storage/storageAccounts@2021-04-01' existing = { - name: last(split(resourceId, '/')) -} - -resource roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for principal in principals: { - name: guid(storage.name, principal.id, roleDefinitionIdOrName) - properties: { - description: description - roleDefinitionId: builtInRoleNames[?roleDefinitionIdOrName] ?? roleDefinitionIdOrName - principalId: principal.id - principalType: !empty(principalType) ? principalType : null - delegatedManagedIdentityResourceId: crossTenant ? principal.resourceId : null - } - scope: storage -}] diff --git a/bicep/modules/storage-account-orig/README.md b/bicep/modules/storage-account-orig/README.md deleted file mode 100644 index 8bcfc8dc..00000000 --- a/bicep/modules/storage-account-orig/README.md +++ /dev/null @@ -1,128 +0,0 @@ -# Azure Storage Module - -This module deploys an Azure Storage Account. - -## Description - -This module supports the following features. - -- Containers -- Tables -- Shares -- Role Assignments -- Diagnostics -- Private Link (Secure network) -- Customer Managed Encryption Keys -- Point in Time Restore -- SAS Secrets - -## Parameters - -| Name | Type | Required | Description | -| :-------------------------------------- | :------: | :------: | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `resourceName` | `string` | Yes | Used to name all resources | -| `location` | `string` | No | Resource Location. | -| `tags` | `object` | No | Tags. | -| `lock` | `string` | No | Optional. Specify the type of lock. | -| `sku` | `string` | No | Specifies the storage account sku type. | -| `accessTier` | `string` | No | Specifies the storage account access tier. | -| `containers` | `array` | No | Optional. Array of Storage Containers to be created. | -| `tables` | `array` | No | Optional. Array of Storage Tables to be created. | -| `shares` | `array` | No | Optional. Array of Storage Shares to be created. | -| `shareQuota` | `int` | No | Optional. The maximum size of the share, in gigabytes. Must be greater than 0, and less than or equal to 5120 (5TB). For Large File Shares, the maximum size is 102400 (100TB). | -| `enabledProtocols` | `string` | No | Optional. The authentication protocol that is used for the file share. Can only be specified when creating a share. | -| `rootSquash` | `string` | No | Optional. Permissions for NFS file shares are enforced by the client OS rather than the Azure Files service. Toggling the root squash behavior reduces the rights of the root user for NFS shares. | -| `crossTenant` | `bool` | No | Optional. Indicates if the module is used in a cross tenant scenario. If true, a resourceId must be provided in the role assignment's principal object. | -| `roleAssignments` | `array` | No | Optional. Array of objects that describe RBAC permissions, format { roleDefinitionResourceId (string), principalId (string), principalType (enum), enabled (bool) }. Ref: https://docs.microsoft.com/en-us/azure/templates/microsoft.authorization/roleassignments?tabs=bicep | -| `diagnosticWorkspaceId` | `string` | No | Optional. Resource ID of the diagnostic log analytics workspace. | -| `diagnosticStorageAccountId` | `string` | No | Optional. Resource ID of the diagnostic storage account. | -| `diagnosticEventHubAuthorizationRuleId` | `string` | No | Optional. Resource ID of the diagnostic event hub authorization rule for the Event Hubs namespace in which the event hub should be created or streamed to. | -| `diagnosticEventHubName` | `string` | No | Optional. Name of the diagnostic event hub within the namespace to which logs are streamed. Without this, an event hub is created for each log category. | -| `diagnosticLogsRetentionInDays` | `int` | No | Optional. Specifies the number of days that logs will be kept for; a value of 0 will retain data indefinitely. | -| `logsToEnable` | `array` | No | Optional. The name of logs that will be streamed. | -| `metricsToEnable` | `array` | No | Optional. The name of metrics that will be streamed. | -| `cmekConfiguration` | `object` | No | Optional. Customer Managed Encryption Key. | -| `deleteRetention` | `int` | No | Amount of days the soft deleted data is stored and available for recovery. 0 is off. | -| `privateLinkSettings` | `object` | No | Settings Required to Enable Private Link | -| `keyVaultName` | `string` | No | Optional: Key Vault Name to store secrets into | -| `storageAccountSecretName` | `string` | No | Optional: To save storage account name into vault set the secret name. | -| `storageAccountKeySecretName` | `string` | No | Optional: To save storage account key into vault set the secret name. | -| `storageAccountConnectionString` | `string` | No | Optional: To save storage account connectionstring into vault set the secret name. | -| `basetime` | `string` | No | Optional: Current Date Time | -| `sasProperties` | `object` | No | Optional: Default SAS TOken Properties to download Blob. | -| `saveToken` | `bool` | No | Optional: To save storage account sas token into vault set the properties. | - -## Outputs - -| Name | Type | Description | -| :--- | :----: | :------------------------ | -| id | string | The resource ID. | -| name | string | The name of the resource. | - -## Examples - -### Example 1 - -```bicep -module storage 'br:osdubicep.azurecr.io/public/storage-account:1.0.8' = { - name: 'storage_account' - params: { - resourceName: resourceName - location: location - sku: 'Standard_LRS' - } -} -``` - -### Example 2 - -```bicep -module storage 'br:osdubicep.azurecr.io/public/storage-account:1.0.8' = { - name: 'storage_account' - params: { - resourceName: resourceName - location: location - sku: 'Standard_LRS' - - containers: [ - 'container1' - 'another' - ] - - tables: [ - 'table1' - 'another' - ] - - // Add Role Assignment - roleAssignments: [ - { - roleDefinitionIdOrName: 'Storage Blob Data Contributor' - principalIds: [ - identity.outputs.principalId - ] - principalType: 'ServicePrincipal' - } - ] - - // Enable Diagnostics - diagnosticWorkspaceId: logs.outputs.id - - // Enable Private Link - privateLinkSettings:{ - vnetId: network.outputs.id - subnetId: network.outputs.subnetIds[0] - } - - // Enable Customer Managed Encryption Key - cmekConfiguration: { - kvUrl: 'https://akeyvault.vault.azure.net' - keyName: 'akey' - identityId: '/subscriptions/222222-2222-2222-2222-2222222222/resourcegroups/agroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aidentity' - } - - // Enable Point in Time Restore - deleteRetention: 7 - } -} -``` \ No newline at end of file diff --git a/bicep/modules/storage-account-orig/main.bicep b/bicep/modules/storage-account-orig/main.bicep deleted file mode 100644 index 69046c69..00000000 --- a/bicep/modules/storage-account-orig/main.bicep +++ /dev/null @@ -1,574 +0,0 @@ -targetScope = 'resourceGroup' - -@description('Used to name all resources') -param name string - -@description('Resource Location.') -param location string = resourceGroup().location - -@description('Tags.') -param tags object = {} - -@allowed([ - 'CanNotDelete' - 'NotSpecified' - 'ReadOnly' -]) -@description('Optional. Specify the type of lock.') -param lock string = 'NotSpecified' - -@description('Specifies the storage account sku type.') -@allowed([ - 'Standard_LRS' - 'Premium_LRS' - 'Standard_GRS' -]) -param sku string = 'Standard_LRS' - -@description('Specifies the storage account access tier.') -@allowed([ - 'Cool' - 'Hot' -]) -param accessTier string = 'Hot' - - -@description('Optional. Array of Storage Containers to be created.') -param containers array = [ - /* example - 'one' - 'two' - */ -] - -@description('Optional. Array of Storage Tables to be created.') -param tables array = [ - /* example - 'one' - 'two' - */ -] - -@description('Optional. Array of Storage Shares to be created.') -param shares array = [ - /* example - 'one' - 'two' - */ -] - -@description('Optional. The maximum size of the share, in gigabytes. Must be greater than 0, and less than or equal to 5120 (5TB). For Large File Shares, the maximum size is 102400 (100TB).') -param shareQuota int = 5120 - -@allowed([ - 'NFS' - 'SMB' -]) -@description('Optional. The authentication protocol that is used for the file share. Can only be specified when creating a share.') -param enabledProtocols string = 'SMB' - -@allowed([ - 'AllSquash' - 'NoRootSquash' - 'RootSquash' -]) -@description('Optional. Permissions for NFS file shares are enforced by the client OS rather than the Azure Files service. Toggling the root squash behavior reduces the rights of the root user for NFS shares.') -param rootSquash string = 'NoRootSquash' - -@description('Optional. Indicates if the module is used in a cross tenant scenario. If true, a resourceId must be provided in the role assignment\'s principal object.') -param crossTenant bool = false - -@description('Optional. Array of objects that describe RBAC permissions, format { roleDefinitionResourceId (string), principalId (string), principalType (enum), enabled (bool) }. Ref: https://docs.microsoft.com/en-us/azure/templates/microsoft.authorization/roleassignments?tabs=bicep') -param roleAssignments array = [ - /* example - { - roleDefinitionIdOrName: 'Reader' - principals: [ - { - id: '222222-2222-2222-2222-2222222222' - resourceId: '/subscriptions/111111-1111-1111-1111-1111111111/resourcegroups/rg-osdu-bicep/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-ManagedIdentityName' - } - ] - principalType: 'ServicePrincipal' - } - */ -] - -@description('Optional. Resource ID of the diagnostic log analytics workspace.') -param diagnosticWorkspaceId string = '' - -@description('Optional. Resource ID of the diagnostic storage account.') -param diagnosticStorageAccountId string = '' - -@description('Optional. Resource ID of the diagnostic event hub authorization rule for the Event Hubs namespace in which the event hub should be created or streamed to.') -param diagnosticEventHubAuthorizationRuleId string = '' - -@description('Optional. Name of the diagnostic event hub within the namespace to which logs are streamed. Without this, an event hub is created for each log category.') -param diagnosticEventHubName string = '' - -@description('Optional. Specifies the number of days that logs will be kept for; a value of 0 will retain data indefinitely.') -@minValue(0) -@maxValue(365) -param diagnosticLogsRetentionInDays int = 365 - -@description('Optional. The name of logs that will be streamed.') -@allowed([ - 'StorageRead' - 'StorageWrite' - 'StorageDelete' -]) -param logsToEnable array = [ - 'StorageRead' - 'StorageWrite' - 'StorageDelete' -] - -@description('Optional. The name of metrics that will be streamed.') -@allowed([ - 'AllMetrics' -]) -param metricsToEnable array = [ - 'AllMetrics' -] - -@description('Optional. Customer Managed Encryption Key.') -param cmekConfiguration object = { - kvUrl: '' - keyName: '' - identityId: '' -} - -@description('Amount of days the soft deleted data is stored and available for recovery. 0 is off.') -@minValue(0) -@maxValue(7) -param deleteRetention int = 0 - -@description('Optional. Indicates whether public access is enabled for all blobs or containers in the storage account. For security reasons, it is recommended to set it to false.') -param allowBlobPublicAccess bool = false - -@description('Optional. Indicates whether shared key access is enabled for the storage account.') -param allowSharedKeyAccess bool = true - -@description('Optional. Default to Microsoft Entra authorization in the Azure portal.') -param defaultToOAuthAuthentication bool = true - -var enableCMEK = !empty(cmekConfiguration.kvUrl) && !empty(cmekConfiguration.keyName) && !empty(cmekConfiguration.identityId) ? true : false - -var diagnosticsLogs = [for log in logsToEnable: { - category: log - enabled: true - retentionPolicy: { - enabled: true - days: diagnosticLogsRetentionInDays - } -}] - -var diagnosticsMetrics = [for metric in metricsToEnable: { - category: metric - timeGrain: null - enabled: true - retentionPolicy: { - enabled: true - days: diagnosticLogsRetentionInDays - } -}] - - - -// Create Storage Account -resource storage 'Microsoft.Storage/storageAccounts@2023-01-01' = { - name: length(name) > 24 ? substring(name, 0, 24) : name - location: location - tags: tags - sku: { - name: sku - } - kind: 'StorageV2' - - identity: enableCMEK ? { - type: 'UserAssigned' - userAssignedIdentities: { - '${cmekConfiguration.identityId}': {} - } - } : null - - properties: { - accessTier: accessTier - minimumTlsVersion: 'TLS1_2' - - encryption: enableCMEK ? { - identity: { - userAssignedIdentity: cmekConfiguration.identityId - } - services: { - blob: { - enabled: true - } - table: { - enabled: true - } - file: { - enabled: true - } - } - keySource: 'Microsoft.Keyvault' - keyvaultproperties: { - keyname: cmekConfiguration.keyName - keyvaulturi: cmekConfiguration.kvUrl - } - } : { - services: { - blob: { - enabled: true - } - table: { - enabled: true - } - file: { - enabled: true - } - } - keySource: 'Microsoft.Storage' - } - - allowBlobPublicAccess: allowBlobPublicAccess - defaultToOAuthAuthentication: defaultToOAuthAuthentication - allowSharedKeyAccess: allowSharedKeyAccess - - networkAcls: enablePrivateLink ? { - bypass: 'AzureServices' - defaultAction: 'Deny' - } : { - bypass: 'AzureServices' - defaultAction: 'Allow' - } - } -} - -resource blobServices 'Microsoft.Storage/storageAccounts/blobServices@2023-01-01' = { - parent: storage - name: 'default' - properties: deleteRetention > 0 ? { - changeFeed: { - enabled: true - } - restorePolicy: { - enabled: true - days: 7 - } - isVersioningEnabled: true - deleteRetentionPolicy: { - enabled: true - days: max(deleteRetention, 1) - } - } : { - deleteRetentionPolicy: { - enabled: false - allowPermanentDelete: false - } - } -} - -resource tableServices 'Microsoft.Storage/storageAccounts/tableServices@2023-01-01' = { - name: 'default' - parent: storage - properties: {} -} - -resource fileServices 'Microsoft.Storage/storageAccounts/fileServices@2023-01-01' = { - name: 'default' - parent: storage - properties: { - protocolSettings: {} - shareDeleteRetentionPolicy: { - enabled: true - days: 7 - } - } -} - -resource storage_containers 'Microsoft.Storage/storageAccounts/blobServices/containers@2023-01-01' = [for item in containers: { - parent: blobServices - name: item - properties: { - defaultEncryptionScope: '$account-encryption-key' - denyEncryptionScopeOverride: false - publicAccess: 'None' - } -}] - -resource storage_tables 'Microsoft.Storage/storageAccounts/tableServices/tables@2023-01-01' = [for item in tables: { - parent: tableServices - name: item -}] - -resource fileShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2023-01-01' = [for item in shares: { - parent: fileServices - name: item - properties: { - shareQuota: shareQuota - rootSquash: enabledProtocols == 'NFS' ? rootSquash : null - enabledProtocols: enabledProtocols - } -}] - -// Apply Resource Lock -resource resource_lock 'Microsoft.Authorization/locks@2020-05-01' = if (lock != 'NotSpecified') { - name: '${storage.name}-${lock}-lock' - properties: { - level: lock - notes: lock == 'CanNotDelete' ? 'Cannot delete resource or child resources.' : 'Cannot modify the resource or child resources.' - } - scope: storage -} - -module storage_rbac '.bicep/nested_rbac.bicep' = [for (roleAssignment, index) in roleAssignments: { - name: '${deployment().name}-rbac-${index}' - params: { - description: roleAssignment.?description ?? '' - principals: roleAssignment.principals - roleDefinitionIdOrName: roleAssignment.roleDefinitionIdOrName - principalType: roleAssignment.?principalType ?? '' - resourceId: storage.id - crossTenant: crossTenant - } -}] - - - -// Hook up Diagnostics -resource storage_diagnostics 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = if (!empty(diagnosticStorageAccountId) || !empty(diagnosticWorkspaceId) || !empty(diagnosticEventHubAuthorizationRuleId) || !empty(diagnosticEventHubName)) { - name: 'storage-diagnostics' - scope: blobServices - properties: { - storageAccountId: !empty(diagnosticStorageAccountId) ? diagnosticStorageAccountId : null - workspaceId: !empty(diagnosticWorkspaceId) ? diagnosticWorkspaceId : null - eventHubAuthorizationRuleId: !empty(diagnosticEventHubAuthorizationRuleId) ? diagnosticEventHubAuthorizationRuleId : null - eventHubName: !empty(diagnosticEventHubName) ? diagnosticEventHubName : null - metrics: diagnosticsMetrics - logs: diagnosticsLogs - } - dependsOn: [ - storage - ] -} - -@description('The resource ID.') -output id string = storage.id - -@description('The name of the resource.') -output name string = storage.name - -//////////////// -// Private Link -//////////////// - -@description('Settings Required to Enable Private Link') -param privateLinkSettings object = { - subnetId: '1' // Specify the Subnet for Private Endpoint - vnetId: '1' // Specify the Virtual Network for Virtual Network Link -} - -var enablePrivateLink = privateLinkSettings.vnetId != '1' && privateLinkSettings.subnetId != '1' - - -@description('Specifies the name of the private link to the Resource.') -var privateEndpointName = '${name}-PrivateEndpoint' - -var publicDNSZoneForwarder = 'blob.${environment().suffixes.storage}' -var privateDnsZoneName = 'privatelink.${publicDNSZoneForwarder}' - -resource privateDNSZone 'Microsoft.Network/privateDnsZones@2020-06-01' = if (enablePrivateLink) { - name: privateDnsZoneName - location: 'global' - properties: {} -} - -resource privateEndpoint 'Microsoft.Network/privateEndpoints@2022-01-01' = if (enablePrivateLink) { - name: privateEndpointName - location: location - properties: { - privateLinkServiceConnections: [ - { - name: privateEndpointName - properties: { - privateLinkServiceId: storage.id - groupIds: [ - 'blob' - ] - } - } - ] - subnet: { - id: privateLinkSettings.subnetId - } - } - dependsOn: [ - storage - ] -} - -resource privateDNSZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2022-01-01' = if (enablePrivateLink) { - parent: privateEndpoint - name: 'dnsgroupname' - properties: { - privateDnsZoneConfigs: [ - { - name: 'dnsConfig' - properties: { - privateDnsZoneId: privateDNSZone.id - } - } - ] - } - dependsOn: [ - privateDNSZone - ] -} - -#disable-next-line BCP081 -resource virtualNetworkLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = if (enablePrivateLink) { - parent: privateDNSZone - name: 'link_to_vnet' - location: 'global' - properties: { - registrationEnabled: false - virtualNetwork: { - id: privateLinkSettings.vnetId - } - } - dependsOn: [ - privateDNSZone - ] -} - -//////////////// -// Secrets -//////////////// - -@description('Optional: Key Vault Name to store secrets into') -param keyVaultName string = '' - -@description('Optional: To save storage account name into vault set the secret name.') -param storageAccountSecretName string = '' - -@description('Optional: To save storage account key into vault set the secret name.') -param storageAccountKeySecretName string = '' - -@description('Optional: To save storage account table endpoint into vault set the secret name.') -param storageAccountTableEndpointSecretName string = '' - -@description('Optional: To save storage account blob endpoint into vault set the secret name.') -param storageAccountBlobEndpointSecretName string = '' - -@description('Optional: To save storage account connectionstring into vault set the secret name.') -param storageAccountConnectionString string = '' - -@description('Optional: Current Date Time') -param basetime string = utcNow('u') - -@description('Optional: Default SAS TOken Properties to download Blob.') -param sasProperties object = { - signedServices: 'b' - signedPermission: 'rl' - signedExpiry: dateTimeAdd(basetime, 'P1Y') - signedResourceTypes: 'sco' - signedProtocol: 'https' -} - -@description('Optional: To save storage account sas token into vault set the properties.') -param saveToken bool = false - -@description('Optional: Enable as System Storage.') -param isSystem bool = false - -module systemSecretStorageAccountName '.bicep/keyvault_secrets.bicep' = if (isSystem) { - name: '${deployment().name}-system-secret-name' - params: { - keyVaultName: keyVaultName - name: 'system-storage' - value: storage.name - } -} - -module systemSecretStorageAccountKey '.bicep/keyvault_secrets.bicep' = if (isSystem) { - name: '${deployment().name}-system-secret-key' - params: { - keyVaultName: keyVaultName - name: 'system-storage-key' - value: storage.listKeys().keys[0].value - } -} - -module systemSecretStorageAccountEndpoint '.bicep/keyvault_secrets.bicep' = if (isSystem) { - name: '${deployment().name}-system-secret-endpoint' - params: { - keyVaultName: keyVaultName - name: 'system-storage-blob-endpoint' - value: storage.properties.primaryEndpoints.blob - } -} - -module systemSecretStorageAccountConnection '.bicep/keyvault_secrets.bicep' = if (isSystem) { - name: '${deployment().name}-system-secret-connectionstring' - params: { - keyVaultName: keyVaultName - name: 'system-storage-connection' - value: 'DefaultEndpointsProtocol=https;AccountName=${storage.name};AccountKey=${storage.listKeys().keys[0].value};EndpointSuffix=${environment().suffixes.storage}' - } -} - -module secretStorageAccountName '.bicep/keyvault_secrets.bicep' = if (!empty(keyVaultName) && !empty(storageAccountSecretName)) { - name: '${deployment().name}-secret-name' - params: { - keyVaultName: keyVaultName - name: storageAccountSecretName - value: storage.name - } -} - -module secretStorageAccountKey '.bicep/keyvault_secrets.bicep' = if (!empty(keyVaultName) && !empty(storageAccountKeySecretName)) { - name: '${deployment().name}-secret-key' - params: { - keyVaultName: keyVaultName - name: storageAccountKeySecretName - value: storage.listKeys().keys[0].value - } -} - - -module secretStorageAccountTableEndpoint '.bicep/keyvault_secrets.bicep' = if (!empty(keyVaultName) && !empty(storageAccountTableEndpointSecretName)) { - name: '${deployment().name}-secret-table-endpoint' - params: { - keyVaultName: keyVaultName - name: storageAccountTableEndpointSecretName - value: storage.properties.primaryEndpoints.table - } -} - -module secretStorageAccountBlobEndpoint '.bicep/keyvault_secrets.bicep' = if (!empty(keyVaultName) && !empty(storageAccountBlobEndpointSecretName)) { - name: '${deployment().name}-secret-blob-endpoint' - params: { - keyVaultName: keyVaultName - name: storageAccountBlobEndpointSecretName - value: storage.properties.primaryEndpoints.blob - } -} - -module secretStorageAccountConnection '.bicep/keyvault_secrets.bicep' = if (!empty(keyVaultName) && !empty(storageAccountConnectionString)) { - name: '${deployment().name}-secret-connectionstring' - params: { - keyVaultName: keyVaultName - name: storageAccountConnectionString - value: 'DefaultEndpointsProtocol=https;AccountName=${storage.name};AccountKey=${storage.listKeys().keys[0].value};EndpointSuffix=${environment().suffixes.storage}' - } -} - -module secretSASToken '.bicep/keyvault_secrets.bicep' = if (!empty(keyVaultName) && saveToken) { - name: '${deployment().name}-secret-sasToken' - params: { - keyVaultName: keyVaultName - name: '${storage.name}-SAS' - value: listAccountSAS(storage.name, '2022-05-01', sasProperties).accountSasToken - } -} diff --git a/bicep/modules/storage-account-orig/metadata.json b/bicep/modules/storage-account-orig/metadata.json deleted file mode 100644 index e1af3154..00000000 --- a/bicep/modules/storage-account-orig/metadata.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "$schema": "https://aka.ms/bicep-registry-module-metadata-file-schema-v2#", - "name": "Azure Storage Module", - "summary": "This module deploys an Azure Storage Account.", - "owner": "azure-global-energy" -} \ No newline at end of file diff --git a/bicep/modules/storage-account-orig/test/main.test.bicep b/bicep/modules/storage-account-orig/test/main.test.bicep deleted file mode 100644 index 804c4e7e..00000000 --- a/bicep/modules/storage-account-orig/test/main.test.bicep +++ /dev/null @@ -1,20 +0,0 @@ -targetScope = 'resourceGroup' - -@minLength(3) -@maxLength(22) -@description('Used to name all resources') -param resourceName string - -@description('Registry Location.') -param location string = resourceGroup().location - - -// Module --> Create Storage Account -module storage '../main.bicep' = { - name: 'storage_account' - params: { - name: 'sa${replace(resourceName, '-', '')}${uniqueString(resourceGroup().id, resourceName)}' - location: location - sku: 'Standard_LRS' - } -} diff --git a/bicep/modules/storage-account-orig/test/parameters.json b/bicep/modules/storage-account-orig/test/parameters.json deleted file mode 100644 index 80279e80..00000000 --- a/bicep/modules/storage-account-orig/test/parameters.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "template": "../main.bicep" - }, - "parameters": { - "resourceName": { - "value": "osdu" - } - } -} \ No newline at end of file diff --git a/bicep/modules/storage-account-orig/version.json b/bicep/modules/storage-account-orig/version.json deleted file mode 100644 index ba0bb44c..00000000 --- a/bicep/modules/storage-account-orig/version.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "$schema": "https://aka.ms/bicep-registry-module-version-file-schema#", - "version": "v1.0", - "pathFilters": [ - "./main.json", - "./metadata.json" - ] -} \ No newline at end of file diff --git a/software/components/airflow/pvc.yaml b/software/components/airflow/pvc.yaml index 53a5a4e7..3bc76ac3 100644 --- a/software/components/airflow/pvc.yaml +++ b/software/components/airflow/pvc.yaml @@ -11,9 +11,15 @@ spec: storage: 5Gi accessModes: - ReadWriteMany - azureFile: - secretName: airflow-secrets - shareName: airflow-logs + csi: + driver: file.csi.azure.com + readOnly: false + volumeAttributes: + secretName: airflow-secrets # No secret needed for managed identity + shareName: airflow-logs + # azureFile: + # secretName: airflow-secrets + # shareName: airflow-logs volumeMode: Filesystem mountOptions: - dir_mode=0777 @@ -52,9 +58,15 @@ spec: storage: 5Gi accessModes: - ReadWriteMany - azureFile: - secretName: airflow-secrets - shareName: airflow-dags + csi: + driver: file.csi.azure.com + readOnly: false + volumeAttributes: + secretName: airflow-secrets # No secret needed for managed identity + shareName: airflow-logs + # azureFile: + # secretName: airflow-secrets + # shareName: airflow-dags volumeMode: Filesystem mountOptions: - dir_mode=0777 diff --git a/tools/rest-scripts/partition.http b/tools/rest-scripts/partition.http index 9165da9b..bfe08101 100644 --- a/tools/rest-scripts/partition.http +++ b/tools/rest-scripts/partition.http @@ -203,12 +203,10 @@ GET {{PARTITION_HOST}}/partitions Authorization: Bearer {{access_token}} Accept: application/json -@partition = {{getPartitions.response.body.$[0]}} - ### # @name getPartition -GET {{PARTITION_HOST}}/partitions/{{partition}} +GET {{PARTITION_HOST}}/partitions/{{getPartitions.response.body.$[0]}} Authorization: Bearer {{access_token}} Content-Type: application/json From 0f5685381f892e54a20ec2829b0782bf7e71f9d5 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 7 Nov 2024 08:15:26 -0600 Subject: [PATCH 02/87] Adding workload identity to airflow. --- software/components/airflow/release.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 105daef7..39fb6fb0 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -38,6 +38,11 @@ spec: repository: apache/airflow tag: 2.10.1-python3.12 executor: KubernetesExecutor + serviceAccount: + create: false + name: workload-identity-sa + annotations: + azure.workload.identity/use: "true" securityContext: fsGroup: 1000 runAsUser: 1000 From 34d685e1e947236c5c45777b0574f195643c3fe6 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 7 Nov 2024 09:13:40 -0600 Subject: [PATCH 03/87] Moved to disks. --- software/components/airflow/pvc.yaml | 189 ++++++++++++----------- software/components/airflow/release.yaml | 10 +- software/components/airflow/storage.yaml | 48 ++++++ 3 files changed, 148 insertions(+), 99 deletions(-) create mode 100644 software/components/airflow/storage.yaml diff --git a/software/components/airflow/pvc.yaml b/software/components/airflow/pvc.yaml index 3bc76ac3..3fa31c6c 100644 --- a/software/components/airflow/pvc.yaml +++ b/software/components/airflow/pvc.yaml @@ -1,94 +1,95 @@ ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: airflow-logs-pv - namespace: airflow - labels: - usage: airflow-logs-pv -spec: - capacity: - storage: 5Gi - accessModes: - - ReadWriteMany - csi: - driver: file.csi.azure.com - readOnly: false - volumeAttributes: - secretName: airflow-secrets # No secret needed for managed identity - shareName: airflow-logs - # azureFile: - # secretName: airflow-secrets - # shareName: airflow-logs - volumeMode: Filesystem - mountOptions: - - dir_mode=0777 - - file_mode=0777 - - uid=50000 - - gid=0 - - mfsymlinks - - nobrl ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: airflow-logs-pvc - namespace: airflow - annotations: - volume.beta.kubernetes.io/storage-class: "" -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 5Gi - selector: - matchLabels: - usage: airflow-logs-pv ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: airflow-dags-pv - namespace: airflow - labels: - usage: airflow-dags-pv -spec: - capacity: - storage: 5Gi - accessModes: - - ReadWriteMany - csi: - driver: file.csi.azure.com - readOnly: false - volumeAttributes: - secretName: airflow-secrets # No secret needed for managed identity - shareName: airflow-logs - # azureFile: - # secretName: airflow-secrets - # shareName: airflow-dags - volumeMode: Filesystem - mountOptions: - - dir_mode=0777 - - file_mode=0777 - - uid=1000 - - gid=1000 - - mfsymlinks - - nobrl ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: airflow-dags-pvc - namespace: airflow - annotations: - volume.beta.kubernetes.io/storage-class: "" -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 5Gi - selector: - matchLabels: - usage: airflow-dags-pv \ No newline at end of file +# --- +# apiVersion: v1 +# kind: PersistentVolume +# metadata: +# name: airflow-logs-pv +# namespace: airflow +# labels: +# usage: airflow-logs-pv +# spec: +# capacity: +# storage: 5Gi +# accessModes: +# - ReadWriteMany +# csi: +# driver: file.csi.azure.com +# volumeHandle: #airflow-logs # Actual Storage account name and share +# readOnly: false +# volumeAttributes: +# # secretName: airflow-secrets # No secret needed for managed identity +# shareName: airflow-logs +# # azureFile: +# # secretName: airflow-secrets +# # shareName: airflow-logs +# volumeMode: Filesystem +# mountOptions: +# - dir_mode=0777 +# - file_mode=0777 +# - uid=50000 +# - gid=0 +# - mfsymlinks +# - nobrl +# --- +# apiVersion: v1 +# kind: PersistentVolumeClaim +# metadata: +# name: airflow-logs-pvc +# namespace: airflow +# annotations: +# volume.beta.kubernetes.io/storage-class: "" +# spec: +# accessModes: +# - ReadWriteMany +# resources: +# requests: +# storage: 5Gi +# selector: +# matchLabels: +# usage: airflow-logs-pv +# --- +# apiVersion: v1 +# kind: PersistentVolume +# metadata: +# name: airflow-dags-pv +# namespace: airflow +# labels: +# usage: airflow-dags-pv +# spec: +# capacity: +# storage: 5Gi +# accessModes: +# - ReadWriteMany +# csi: +# driver: file.csi.azure.com +# readOnly: false +# volumeAttributes: +# secretName: airflow-secrets # No secret needed for managed identity +# shareName: airflow-logs +# azureFile: +# secretName: airflow-secrets +# shareName: airflow-dags +# volumeMode: Filesystem +# mountOptions: +# - dir_mode=0777 +# - file_mode=0777 +# - uid=1000 +# - gid=1000 +# - mfsymlinks +# - nobrl +# --- +# apiVersion: v1 +# kind: PersistentVolumeClaim +# metadata: +# name: airflow-dags-pvc +# namespace: airflow +# annotations: +# volume.beta.kubernetes.io/storage-class: "" +# spec: +# accessModes: +# - ReadWriteMany +# resources: +# requests: +# storage: 5Gi +# selector: +# matchLabels: +# usage: airflow-dags-pv \ No newline at end of file diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 39fb6fb0..bd0b42fd 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -38,11 +38,11 @@ spec: repository: apache/airflow tag: 2.10.1-python3.12 executor: KubernetesExecutor - serviceAccount: - create: false - name: workload-identity-sa - annotations: - azure.workload.identity/use: "true" + # serviceAccount: + # create: false + # name: workload-identity-sa + # annotations: + # azure.workload.identity/use: "true" securityContext: fsGroup: 1000 runAsUser: 1000 diff --git a/software/components/airflow/storage.yaml b/software/components/airflow/storage.yaml new file mode 100644 index 00000000..dc0ad85e --- /dev/null +++ b/software/components/airflow/storage.yaml @@ -0,0 +1,48 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: azurefile-csi +provisioner: file.csi.azure.com +parameters: + skuName: Standard_LRS +allowVolumeExpansion: true +mountOptions: + - dir_mode=0777 + - file_mode=0777 + - uid=1000 + - gid=1000 + - mfsymlinks + - nobrl +reclaimPolicy: Delete +volumeBindingMode: Immediate +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: airflow-logs-pvc + namespace: airflow + annotations: + csi.storage.k8s.io/share-name: "airflow-logs" +spec: + accessModes: + - ReadWriteMany + storageClassName: azurefile-csi + resources: + requests: + storage: 5Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: airflow-logs-dags + namespace: airflow + annotations: + csi.storage.k8s.io/share-name: "airflow-dags" +spec: + accessModes: + - ReadWriteMany + storageClassName: azurefile-csi + resources: + requests: + storage: 5Gi \ No newline at end of file From 2613374353e90bf76de0459762a6b1f7199c8e99 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 7 Nov 2024 09:23:05 -0600 Subject: [PATCH 04/87] Fixed storage class --- software/components/airflow/storage.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/software/components/airflow/storage.yaml b/software/components/airflow/storage.yaml index dc0ad85e..a7976f26 100644 --- a/software/components/airflow/storage.yaml +++ b/software/components/airflow/storage.yaml @@ -2,7 +2,7 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: azurefile-csi + name: airflow-storage provisioner: file.csi.azure.com parameters: skuName: Standard_LRS @@ -27,7 +27,7 @@ metadata: spec: accessModes: - ReadWriteMany - storageClassName: azurefile-csi + storageClassName: airflow-storage resources: requests: storage: 5Gi @@ -35,14 +35,14 @@ spec: apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: airflow-logs-dags + name: airflow-dags-pvc namespace: airflow annotations: csi.storage.k8s.io/share-name: "airflow-dags" spec: accessModes: - ReadWriteMany - storageClassName: azurefile-csi + storageClassName: airflow-storage resources: requests: storage: 5Gi \ No newline at end of file From f84dcce80f8af5bda8e08e81c57d41c7831ff32d Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 7 Nov 2024 11:18:31 -0600 Subject: [PATCH 05/87] Adding airflow dag job copy. --- bicep/main.bicep | 33 +++--- .../templates/dag-job.yaml | 110 ++++++++++++++++++ charts/osdu-developer-base/values.yaml | 11 +- software/components/airflow/dag-job.yaml | 40 +++++++ software/components/airflow/pvc.yaml | 95 --------------- software/components/airflow/release.yaml | 2 +- .../components/airflow/vault-secrets.yaml | 4 - 7 files changed, 179 insertions(+), 116 deletions(-) create mode 100644 charts/osdu-developer-base/templates/dag-job.yaml create mode 100644 software/components/airflow/dag-job.yaml delete mode 100644 software/components/airflow/pvc.yaml diff --git a/bicep/main.bicep b/bicep/main.bicep index 717b64eb..244df5ed 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -383,6 +383,7 @@ module clusterBlade 'modules/blade_cluster.bicep' = { ] } + /* __________ ___ .___________. _______ .__ __. _______. __ ______ .__ __. | ____\ \ / / | || ____|| \ | | / || | / __ \ | \ | | @@ -418,6 +419,7 @@ module fluxExtension 'modules/flux-extension/main.bicep' = { ] } + /* _______. ______ .______ __ .______ .___________. / | / || _ \ | | | _ \ | | @@ -478,7 +480,6 @@ module extensionClientId 'br/public:avm/res/resources/deployment-script:0.4.0' = | |\ \----.| |____ | |__| | | | .----) | | | | |\ \----. | | | _| `._____||_______| \______| |__| |_______/ |__| | _| `._____| |__| */ - module registry 'br/public:avm/res/container-registry/registry:0.1.1' = { name: '${configuration.name}-container-registry' params: { @@ -520,6 +521,7 @@ module registry 'br/public:avm/res/container-registry/registry:0.1.1' = { } } + /* __ ___ ___________ ____ ____ ____ ___ __ __ __ .___________. | |/ / | ____\ \ / / \ \ / / / \ | | | | | | | | @@ -697,6 +699,7 @@ var commonLayerConfig = { } } + /* _______.___________. ______ .______ ___ _______ _______ / | | / __ \ | _ \ / \ / _____|| ____| | (----`---| |----`| | | | | |_) | / ^ \ | | __ | |__ @@ -1109,20 +1112,20 @@ module configBlade 'modules/blade_configuration.bicep' = { ] } -module storageAcl 'modules/network_acl_storage.bicep' = { - name: '${configuration.name}-storage-acl' - params: { - storageName: storage.outputs.name - location: location - skuName: configuration.storage.sku - natClusterIP: clusterBlade.outputs.natClusterIP - } - dependsOn: [ - manifestDagShareUpload - csvDagShareUpload - gitOpsUpload - ] -} +// module storageAcl 'modules/network_acl_storage.bicep' = { +// name: '${configuration.name}-storage-acl' +// params: { +// storageName: storage.outputs.name +// location: location +// skuName: configuration.storage.sku +// natClusterIP: clusterBlade.outputs.natClusterIP +// } +// dependsOn: [ +// manifestDagShareUpload +// csvDagShareUpload +// gitOpsUpload +// ] +// } // =============== // // Outputs // diff --git a/charts/osdu-developer-base/templates/dag-job.yaml b/charts/osdu-developer-base/templates/dag-job.yaml new file mode 100644 index 00000000..4d458d35 --- /dev/null +++ b/charts/osdu-developer-base/templates/dag-job.yaml @@ -0,0 +1,110 @@ +{{- if .Values.airflow.enabled -}} +{{- range $index, $item := .Values.airflow.items }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $.Release.Name }}-file-upload-{{ $index }} + namespace: {{ $.Release.Namespace }} +spec: + ttlSecondsAfterFinished: 300 + template: + spec: + serviceAccountName: workload-identity-sa + volumes: + - name: script + configMap: + name: file-upload-script-{{ $.Release.Name }} + defaultMode: 0500 + - name: share-storage + persistentVolumeClaim: + claimName: {{ $item.pvc }} + initContainers: + - name: file-upload + image: mcr.microsoft.com/cbl-mariner/base/core:2.0 + command: ["/bin/sh"] + args: + - -c + - | + tdnf install -y curl tar zip && \ + /script/init.sh + volumeMounts: + - name: script + mountPath: "/script" + - name: share-storage + mountPath: "/share" + env: + - name: URL + value: {{ $item.url | quote }} + - name: SHARE + value: {{ $item.name | quote }} + - name: COMPRESS + value: {{ $item.compress | default false | quote }} + - name: FILE + value: {{ $item.folder | quote }} + containers: + - name: completion + image: istio/base + command: ["/bin/sleep", "30"] + volumeMounts: + - name: script + mountPath: "/script" + restartPolicy: Never +--- +{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: file-upload-script-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} +data: + init.sh: | + #!/bin/bash + set -e + + url_basename=$(basename ${URL}) + echo "Derived filename from URL: ${url_basename}" + + echo "Downloading file from ${URL} to ${url_basename}" + retry_count=0 + max_retries=3 + while [ $retry_count -lt $max_retries ]; do + if curl -kso ${url_basename} ${URL}; then + break + else + retry_count=$((retry_count + 1)) + echo "Attempt $retry_count failed. Retrying in 5 seconds..." >&2 + sleep 5 + fi + done + + if [ $retry_count -eq $max_retries ]; then + echo "Error: Failed to download file from ${URL} after $max_retries attempts." >&2 + exit 1 + fi + + mkdir -p /share/${SHARE} + + if [[ ${URL} == *.tar.gz ]]; then + echo "Processing tar.gz archive..." + mkdir -p extracted_files + tar -xzf ${url_basename} --strip-components=1 -C extracted_files + + if [[ ${COMPRESS} == "true" ]]; then + echo "Creating zip archive..." + rm ${url_basename} + zip_filename="${url_basename%.tar.gz}.zip" + original_dir=$(pwd) + cd extracted_files/${FILE} + zip -r ${original_dir}/${zip_filename} * + cd ${original_dir} + cp ${zip_filename} /share/${SHARE}/ + echo "Zip file ${zip_filename} copied to share." + else + echo "Copying extracted files..." + cp -r extracted_files/${FILE} /share/${SHARE}/ + fi + else + echo "Copying single file..." + cp ${FILE} /share/${SHARE}/ + fi +{{- end }} \ No newline at end of file diff --git a/charts/osdu-developer-base/values.yaml b/charts/osdu-developer-base/values.yaml index c085a581..0cd84445 100644 --- a/charts/osdu-developer-base/values.yaml +++ b/charts/osdu-developer-base/values.yaml @@ -45,4 +45,13 @@ share: - name: item2 url: "https://example.com/archive.tar.gz" file: "folder_in_archive" - compress: true \ No newline at end of file + compress: true + +airflow: + enabled: false + items: + - name: manifest + folder: "src/osdu_dags" + compress: true + url: "https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-dags/-/archive/master/ingestion-dags-master.tar.gz" + pvc: "airflow-dags-pvc" \ No newline at end of file diff --git a/software/components/airflow/dag-job.yaml b/software/components/airflow/dag-job.yaml new file mode 100644 index 00000000..aa0143fb --- /dev/null +++ b/software/components/airflow/dag-job.yaml @@ -0,0 +1,40 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: base-airflow-dags + namespace: default + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + targetNamespace: airflow + releaseName: airflow-dags + dependsOn: + - name: azure-keyvault-airflow + namespace: default + chart: + spec: + chart: ./charts/osdu-developer-base + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: config-map-values + valuesKey: values.yaml + values: + azure: + enabled: true + airflow: + enabled: true + items: + - name: manifest + folder: "src/osdu_dags" + compress: true + url: "https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-dags/-/archive/master/ingestion-dags-master.tar.gz" + pvc: "airflow-dags-pvc" \ No newline at end of file diff --git a/software/components/airflow/pvc.yaml b/software/components/airflow/pvc.yaml deleted file mode 100644 index 3fa31c6c..00000000 --- a/software/components/airflow/pvc.yaml +++ /dev/null @@ -1,95 +0,0 @@ -# --- -# apiVersion: v1 -# kind: PersistentVolume -# metadata: -# name: airflow-logs-pv -# namespace: airflow -# labels: -# usage: airflow-logs-pv -# spec: -# capacity: -# storage: 5Gi -# accessModes: -# - ReadWriteMany -# csi: -# driver: file.csi.azure.com -# volumeHandle: #airflow-logs # Actual Storage account name and share -# readOnly: false -# volumeAttributes: -# # secretName: airflow-secrets # No secret needed for managed identity -# shareName: airflow-logs -# # azureFile: -# # secretName: airflow-secrets -# # shareName: airflow-logs -# volumeMode: Filesystem -# mountOptions: -# - dir_mode=0777 -# - file_mode=0777 -# - uid=50000 -# - gid=0 -# - mfsymlinks -# - nobrl -# --- -# apiVersion: v1 -# kind: PersistentVolumeClaim -# metadata: -# name: airflow-logs-pvc -# namespace: airflow -# annotations: -# volume.beta.kubernetes.io/storage-class: "" -# spec: -# accessModes: -# - ReadWriteMany -# resources: -# requests: -# storage: 5Gi -# selector: -# matchLabels: -# usage: airflow-logs-pv -# --- -# apiVersion: v1 -# kind: PersistentVolume -# metadata: -# name: airflow-dags-pv -# namespace: airflow -# labels: -# usage: airflow-dags-pv -# spec: -# capacity: -# storage: 5Gi -# accessModes: -# - ReadWriteMany -# csi: -# driver: file.csi.azure.com -# readOnly: false -# volumeAttributes: -# secretName: airflow-secrets # No secret needed for managed identity -# shareName: airflow-logs -# azureFile: -# secretName: airflow-secrets -# shareName: airflow-dags -# volumeMode: Filesystem -# mountOptions: -# - dir_mode=0777 -# - file_mode=0777 -# - uid=1000 -# - gid=1000 -# - mfsymlinks -# - nobrl -# --- -# apiVersion: v1 -# kind: PersistentVolumeClaim -# metadata: -# name: airflow-dags-pvc -# namespace: airflow -# annotations: -# volume.beta.kubernetes.io/storage-class: "" -# spec: -# accessModes: -# - ReadWriteMany -# resources: -# requests: -# storage: 5Gi -# selector: -# matchLabels: -# usage: airflow-dags-pv \ No newline at end of file diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index bd0b42fd..faa22e71 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -10,7 +10,7 @@ spec: targetNamespace: airflow releaseName: airflow dependsOn: - - name: azure-keyvault-airflow + - name: base-airflow-dags namespace: default install: remediation: diff --git a/software/components/airflow/vault-secrets.yaml b/software/components/airflow/vault-secrets.yaml index 2e64524c..5d85173d 100644 --- a/software/components/airflow/vault-secrets.yaml +++ b/software/components/airflow/vault-secrets.yaml @@ -32,10 +32,6 @@ spec: vaultSecret: airflow-fernet-key - key: webserver-key vaultSecret: airflow-webserver-key - - key: azurestorageaccountname - vaultSecret: system-storage - - key: azurestorageaccountkey - vaultSecret: system-storage-key - key: password vaultSecret: airflow-admin-password - key: connection From 6f71272d6589c74d8677ab2a34644176666f1030 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 7 Nov 2024 11:45:47 -0600 Subject: [PATCH 06/87] Feature flag for nsg lockdown. --- bicep/main.bicep | 2 ++ bicep/main.parameters.json | 3 ++- bicep/modules/blade_cluster.bicep | 5 ++++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/bicep/main.bicep b/bicep/main.bicep index 244df5ed..d50421ac 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -53,6 +53,7 @@ param experimentalSoftware object = { param clusterConfiguration object = { enableNodeAutoProvisioning: true enablePrivateCluster: false + enableLockDown: false } @description('Optional. Bring your own Virtual Network.') @@ -368,6 +369,7 @@ module clusterBlade 'modules/blade_cluster.bicep' = { enableNodeAutoProvisioning: clusterConfiguration.enableNodeAutoProvisioning == 'false' ? false : true enablePrivateCluster: clusterConfiguration.enablePrivateCluster == 'true' ? true : false + nodeResourceGroupLockDown: clusterConfiguration.enableLockDown == 'false' ? false : true workspaceResourceId: logAnalytics.outputs.resourceId identityId: enableVnetInjection ? networkBlade.outputs.networkConfiguration.identityId : stampIdentity.outputs.resourceId diff --git a/bicep/main.parameters.json b/bicep/main.parameters.json index 71468ed1..5db650e1 100644 --- a/bicep/main.parameters.json +++ b/bicep/main.parameters.json @@ -32,7 +32,8 @@ "clusterConfiguration": { "value": { "enableNodeAutoProvisioning": "${ENABLE_NODE_AUTO_PROVISIONING}", - "enablePrivateCluster": "${ENABLE_PRIVATE_CLUSTER}" + "enablePrivateCluster": "${ENABLE_PRIVATE_CLUSTER}", + "enableLockDown": "${ENABLE_LOCK_DOWN}" } }, "vnetConfiguration": { diff --git a/bicep/modules/blade_cluster.bicep b/bicep/modules/blade_cluster.bicep index 9f94eb31..444b0277 100644 --- a/bicep/modules/blade_cluster.bicep +++ b/bicep/modules/blade_cluster.bicep @@ -48,6 +48,9 @@ param enableNodeAutoProvisioning bool = true @description('Feature Flag to Enable Private Cluster') param enablePrivateCluster bool = true +@description('Feature Flag to Enable Node Resource Group Lock Down') +param nodeResourceGroupLockDown bool = true + ///////////////////////////////// // Configuration ///////////////////////////////// @@ -148,7 +151,7 @@ module cluster './managed-cluster/main.bicep' = { disableLocalAccounts: true enableRBAC: true aadProfileManaged: true - nodeResourceGroupLockDown: true + nodeResourceGroupLockDown: nodeResourceGroupLockDown // Observability Settings enableAzureDefender: true From 56b0c2809f8f28ec5a712bf838211741ac1f1f1b Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 7 Nov 2024 13:44:44 -0600 Subject: [PATCH 07/87] Adding CSV DAG --- bicep/main.bicep | 35 +-- bicep/modules/blade_cluster.bicep | 2 - bicep/modules/blade_configuration.bicep | 21 +- .../templates/csv-dag-job.yaml | 207 ++++++++++++++++++ .../{dag-job.yaml => manifest-dag-job.yaml} | 8 +- software/components/airflow/dag-job.yaml | 35 ++- 6 files changed, 271 insertions(+), 37 deletions(-) create mode 100644 charts/osdu-developer-base/templates/csv-dag-job.yaml rename charts/osdu-developer-base/templates/{dag-job.yaml => manifest-dag-job.yaml} (94%) diff --git a/bicep/main.bicep b/bicep/main.bicep index d50421ac..ac08a435 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -948,24 +948,24 @@ module gitOpsUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for //TODO: This needs to be removed and moved into a kubernetes job. -module manifestDagShareUpload 'modules/script-share-upload/main.bicep' = { - name: '${configuration.name}-storage-dag-upload-manifest' - params: { - newStorageAccount: true - location: location - storageAccountName: storage.outputs.name - identityName: stampIdentity.outputs.name +// module manifestDagShareUpload 'modules/script-share-upload/main.bicep' = { +// name: '${configuration.name}-storage-dag-upload-manifest' +// params: { +// newStorageAccount: true +// location: location +// storageAccountName: storage.outputs.name +// identityName: stampIdentity.outputs.name - shareName: 'airflow-dags' - filename: 'src/osdu_dags' - compress: true - fileurl: 'https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-dags/-/archive/master/ingestion-dags-master.tar.gz' - } - dependsOn: [ - stampIdentity - storage - ] -} +// shareName: 'airflow-dags' +// filename: 'src/osdu_dags' +// compress: true +// fileurl: 'https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-dags/-/archive/master/ingestion-dags-master.tar.gz' +// } +// dependsOn: [ +// stampIdentity +// storage +// ] +// } //TODO: This needs to be removed and moved into a kubernetes job. module csvDagShareUpload 'modules/script-share-csvdag/main.bicep' = { @@ -1081,6 +1081,7 @@ module configBlade 'modules/blade_configuration.bicep' = { managedIdentityName: stampIdentity.outputs.name kvName: keyvault.outputs.name kvUri: keyvault.outputs.uri + appInsightsKey: insights.outputs.instrumentationKey partitionStorageNames: partitionBlade.outputs.partitionStorageNames partitionServiceBusNames: partitionBlade.outputs.partitionServiceBusNames diff --git a/bicep/modules/blade_cluster.bicep b/bicep/modules/blade_cluster.bicep index 444b0277..df41ae9e 100644 --- a/bicep/modules/blade_cluster.bicep +++ b/bicep/modules/blade_cluster.bicep @@ -381,8 +381,6 @@ module appConfigExtension './managed-cluster/aks_appconfig_extension.bicep' = { - - // =============== // // Outputs // // =============== // diff --git a/bicep/modules/blade_configuration.bicep b/bicep/modules/blade_configuration.bicep index 3327ac74..340d0ca3 100644 --- a/bicep/modules/blade_configuration.bicep +++ b/bicep/modules/blade_configuration.bicep @@ -32,6 +32,9 @@ param applicationClientId string @description('Specify the AD Application Principal Id.') param applicationClientPrincipalOid string = '' +@description('Specify the Application Insights Key.') +param appInsightsKey string + @description('Software GIT Repository URL') param softwareRepository string @@ -98,14 +101,6 @@ param appSettings appConfigItem[] param dateStamp string = utcNow() - -///////////////////////////////// -// Configuration -///////////////////////////////// - - - - ///////////////////////////////// // Existing Resources ///////////////////////////////// @@ -367,13 +362,14 @@ values.yaml: | configEndpoint: {2} keyvaultUri: {3} keyvaultName: {4} - appId: {5} - appOid: {6} + appInsightsKey: {5} + appId: {6} + appOid: {7} ingress: internalGateway: - enabled: {7} - externalGateway: enabled: {8} + externalGateway: + enabled: {9} ''' } @@ -406,6 +402,7 @@ module appConfigMap './aks-config-map/main.bicep' = { app_config.outputs.endpoint, kvUri, kvName, + appInsightsKey, applicationClientId, applicationClientPrincipalOid, clusterIngress == 'Internal' || clusterIngress == 'Both' ? 'true' : 'false', diff --git a/charts/osdu-developer-base/templates/csv-dag-job.yaml b/charts/osdu-developer-base/templates/csv-dag-job.yaml new file mode 100644 index 00000000..014706d8 --- /dev/null +++ b/charts/osdu-developer-base/templates/csv-dag-job.yaml @@ -0,0 +1,207 @@ +{{- if .Values.airflow.enabled -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-csvdag-upload + namespace: {{ .Release.Namespace }} +spec: + ttlSecondsAfterFinished: 300 + template: + spec: + serviceAccountName: workload-identity-sa + volumes: + - name: script + configMap: + name: csvdag-upload-script-{{ .Release.Name }} + defaultMode: 0500 + - name: share-storage + persistentVolumeClaim: + claimName: {{ .Values.airflow.csvdag.pvc }} + initContainers: + - name: csvdag-upload + image: mcr.microsoft.com/cbl-mariner/base/core:2.0 + command: ["/bin/sh"] + args: + - -c + - | + tdnf install -y curl tar zip jq && \ + /script/csvdag.sh + volumeMounts: + - name: script + mountPath: "/script" + - name: share-storage + mountPath: "/share" + env: + - name: URL + value: {{ .Values.airflow.csvdag.url | quote }} + - name: FILE + value: {{ .Values.airflow.csvdag.folder | quote }} + - name: SEARCH_AND_REPLACE + value: {{ .Values.airflow.csvdag.replacements | toJson | quote }} + - name: SEARCH_AND_REPLACE + value: | + [ + {"find":"{| DAG_NAME |}", "replace":"csv-parser"}, + {"find":"{| DOCKER_IMAGE |}", "replace":"community.opengroup.org:5555/osdu/platform/data-flow/ingestion/csv-parser/csv-parser/csv-parser-v0-27-0-azure-1:60747714ac490be0defe8f3e821497b3cce03390"}, + {"find":"{| NAMESPACE |}", "replace":"airflow"}, + {"find":"{| K8S_POD_OPERATOR_KWARGS or {} |}", "replace":{"labels":{"aadpodidbinding":"osdu-identity"},"annotations":{"sidecar.istio.io/inject":"false"}}}, + {"find":"{| ENV_VARS or {} |}", "replace":{ + "storage_service_endpoint": "http://storage.osdu-core.svc.cluster.local/api/storage/v2", + "schema_service_endpoint": "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1", + "search_service_endpoint": "http://search.osdu-core.svc.cluster.local/api/search/v2", + "partition_service_endpoint": "http://partition.osdu-core.svc.cluster.local/api/partition/v1", + "unit_service_endpoint": "http://unit.osdu-core.svc.cluster.local/api/unit/v2/unit/symbol", + "file_service_endpoint": "http://file.osdu-core.svc.cluster.local/api/file/v2", + "azure_paas_podidentity_isEnabled": "false", + "KEYVAULT_URI": {{ .Values.azure.keyvaultUri | quote }}, + "AZURE_TENANT_ID": {{ .Values.azure.tenantId | quote }}, + "AZURE_CLIENT_ID": {{ .Values.azure.clientId | quote }}, + "AZURE_CLIENT_SECRET": "your-client-secret", + "appinsights_key": {{ .Values.azure.keyvaultUri | quote }}, + "aad_client_id": {{ .Values.azure.clientId | quote }} + }} + ] + containers: + - name: completion + image: istio/base + command: ["/bin/sleep", "30"] + volumeMounts: + - name: script + mountPath: "/script" + restartPolicy: Never +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: csvdag-upload-script-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} +data: + csvdag.sh: | + #!/bin/bash + set -e + + # This script performs the following tasks: + # 1. Waits for Identity RBAC replication. + # 2. Downloads a tar.gz file from a specified URL and extracts its contents. + # 3. Processes a specific Python file within the extracted contents, performing complex find/replace operations based on a provided JSON configuration. + # 4. Compresses the DAG and copies it to the shared volume. + + echo "Waiting on Identity RBAC replication (${initialDelay})" + sleep "${initialDelay}" + + echo "###########################" + echo "${SEARCH_AND_REPLACE}" + echo "###########################" + + # Download the source code and extract it. + url_basename=$(basename "${URL}") + echo "Derived filename from URL: ${url_basename}" + echo "Downloading file from ${URL} to ${url_basename}" + curl -so "${url_basename}" "${URL}" + echo "Extracting tar.gz archive..." + mkdir -p extracted_files + tar -xzf "${url_basename}" --strip-components=1 -C extracted_files + + # Process the replacements + csv_file="extracted_files/${FILE}/csv_ingestion_all_steps.py" + output_file="extracted_files/${FILE}/csv-parser.py" + + if [ -f "${csv_file}" ]; then + echo "Processing ${csv_file} file" + + # Number of replacements + num_replacements=$(echo "${SEARCH_AND_REPLACE}" | jq '. | length') + + # Initialize arrays + declare -a finds + declare -a replaces + declare -a replace_types + + # Build arrays + for (( idx=0; idx<${num_replacements}; idx++ )); do + finds[$idx]=$(echo "${SEARCH_AND_REPLACE}" | jq -r ".[$idx].find") + replace_type=$(echo "${SEARCH_AND_REPLACE}" | jq -r ".[$idx].replace | type") + replace_types[$idx]=$replace_type + if [ "$replace_type" == "string" ]; then + replaces[$idx]=$(echo "${SEARCH_AND_REPLACE}" | jq -r ".[$idx].replace") + else + replaces[$idx]=$(echo "${SEARCH_AND_REPLACE}" | jq -c ".[$idx].replace") + fi + done + + # Empty the output file + > "$output_file" + + # Read the input file line by line + while IFS= read -r line || [[ -n "$line" ]]; do + replaced=0 + # For each 'find'/'replace' pair + for idx in "${!finds[@]}"; do + find_placeholder="${finds[$idx]}" + replace_value="${replaces[$idx]}" + replace_type="${replace_types[$idx]}" + + if [[ "$line" == *"$find_placeholder"* ]]; then + # Line contains the placeholder + + if [ "$replace_type" == "object" ]; then + # 'replace_value' is a JSON object + + # Split the line at the placeholder + line_before_placeholder="${line%%$find_placeholder*}" + line_after_placeholder="${line#*$find_placeholder}" + + # Get the indentation of the line up to the placeholder + leading_spaces=$(echo "$line_before_placeholder" | sed -n 's/^\(\s*\).*$/\1/p') + + # Format the JSON with jq + formatted_json=$(echo "$replace_value" | jq '.') + + # Indent the JSON + indented_json=$(echo "$formatted_json" | sed "s/^/${leading_spaces}/") + + # Output the line before the placeholder (excluding placeholder) + echo -n "$line_before_placeholder" >> "$output_file" + + # Output the indented JSON + echo "$indented_json" >> "$output_file" + + # Output the rest of the line after the placeholder, if any + if [ -n "$line_after_placeholder" ]; then + echo "$line_after_placeholder" >> "$output_file" + fi + else + # 'replace_value' is a string + + # Replace the placeholder in the line + replaced_line="${line//$find_placeholder/$replace_value}" + + # Output the modified line + echo "$replaced_line" >> "$output_file" + fi + replaced=1 + break # Skip checking other placeholders for this line + fi + done + if [[ $replaced -eq 0 ]]; then + # Line did not contain any placeholder + echo "$line" >> "$output_file" + fi + done < "$csv_file" + + # Remove the original file + rm "$csv_file" + fi + + # Compress the DAG folder and copy it to the shared volume + rm "${url_basename}" + zip_filename="${url_basename%.tar.gz}.zip" + current_dir=$(pwd) + cd "extracted_files/${FILE}" || exit 1 + zip -r "${current_dir}/${zip_filename}" . + cd - || exit 1 + + # Copy the zip file to the shared volume + cp "${zip_filename}" /share/ + echo "Zip file ${zip_filename} copied to shared volume." +{{- end }} \ No newline at end of file diff --git a/charts/osdu-developer-base/templates/dag-job.yaml b/charts/osdu-developer-base/templates/manifest-dag-job.yaml similarity index 94% rename from charts/osdu-developer-base/templates/dag-job.yaml rename to charts/osdu-developer-base/templates/manifest-dag-job.yaml index 4d458d35..1da20c19 100644 --- a/charts/osdu-developer-base/templates/dag-job.yaml +++ b/charts/osdu-developer-base/templates/manifest-dag-job.yaml @@ -82,8 +82,6 @@ data: exit 1 fi - mkdir -p /share/${SHARE} - if [[ ${URL} == *.tar.gz ]]; then echo "Processing tar.gz archive..." mkdir -p extracted_files @@ -97,14 +95,14 @@ data: cd extracted_files/${FILE} zip -r ${original_dir}/${zip_filename} * cd ${original_dir} - cp ${zip_filename} /share/${SHARE}/ + cp ${zip_filename} /share/ echo "Zip file ${zip_filename} copied to share." else echo "Copying extracted files..." - cp -r extracted_files/${FILE} /share/${SHARE}/ + cp -r extracted_files/${FILE} /share/ fi else echo "Copying single file..." - cp ${FILE} /share/${SHARE}/ + cp ${FILE} /share/ fi {{- end }} \ No newline at end of file diff --git a/software/components/airflow/dag-job.yaml b/software/components/airflow/dag-job.yaml index aa0143fb..a0644d40 100644 --- a/software/components/airflow/dag-job.yaml +++ b/software/components/airflow/dag-job.yaml @@ -37,4 +37,37 @@ spec: folder: "src/osdu_dags" compress: true url: "https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-dags/-/archive/master/ingestion-dags-master.tar.gz" - pvc: "airflow-dags-pvc" \ No newline at end of file + pvc: "airflow-dags-pvc" + # csvdag: + # folder: "airflowdags" + # compress: true + # url: "https://community.opengroup.org/osdu/platform/data-flow/ingestion/csv-parser/csv-parser/-/archive/master/csv-parser-master.tar.gz" + # pvc: "airflow-dags-pvc" + # replacements: + # - find: "{| DAG_NAME |}" + # replace: "csv-parser" + # - find: "{| DOCKER_IMAGE |}" + # replace: "community.opengroup.org:5555/osdu/platform/data-flow/ingestion/csv-parser/csv-parser/csv-parser-v0-27-0-azure-1:60747714ac490be0defe8f3e821497b3cce03390" + # - find: "{| NAMESPACE |}" + # replace: "airflow" + # - find: "{| K8S_POD_OPERATOR_KWARGS or {} |}" + # replace: + # labels: + # aadpodidbinding: "osdu-identity" + # annotations: + # "sidecar.istio.io/inject": "false" + # - find: "{| ENV_VARS or {} |}" + # replace: + # storage_service_endpoint: "http://storage.osdu-core.svc.cluster.local/api/storage/v2" + # schema_service_endpoint: "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1" + # search_service_endpoint: "http://search.osdu-core.svc.cluster.local/api/search/v2" + # partition_service_endpoint: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" + # unit_service_endpoint: "http://unit.osdu-core.svc.cluster.local/api/unit/v2/unit/symbol" + # file_service_endpoint: "http://file.osdu-core.svc.cluster.local/api/file/v2" + # KEYVAULT_URI: {{ .Values.azure.keyvaultUri }} + # appinsights_key: {{ .Values.azure.appInsightsKey | default "your-insights-key" }} + # azure_paas_podidentity_isEnabled: "false" + # AZURE_TENANT_ID: {{ .Values.azure.tenantId }} + # AZURE_CLIENT_ID: {{ .Values.azure.clientId }} + # AZURE_CLIENT_SECRET: {{ .Values.azure.clientSecret | default "your-client-secret" }} + # aad_client_id: {{ .Values.azure.clientId }} \ No newline at end of file From a970f8ffd10078e24807042eaeefa01ed71da718 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 7 Nov 2024 14:50:41 -0600 Subject: [PATCH 08/87] Updated --- README.md | 2 +- .../{csv-dag-job.yaml => dag-csv-job.yaml} | 2 +- ...est-dag-job.yaml => dag-manifest-job.yaml} | 4 +- software/components/airflow/dag-job.yaml | 73 ------------------ software/components/airflow/dag-jobs.yaml | 75 +++++++++++++++++++ 5 files changed, 79 insertions(+), 77 deletions(-) rename charts/osdu-developer-base/templates/{csv-dag-job.yaml => dag-csv-job.yaml} (99%) rename charts/osdu-developer-base/templates/{manifest-dag-job.yaml => dag-manifest-job.yaml} (96%) delete mode 100644 software/components/airflow/dag-job.yaml create mode 100644 software/components/airflow/dag-jobs.yaml diff --git a/README.md b/README.md index bc6b731a..c6e8d061 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # OSDU Developer -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) ![GitHub milestone details](https://img.shields.io/github/milestones/progress/azure/osdu-developer/1) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) ![GitHub milestone details](https://img.shields.io/github/milestones/progress/azure/osdu-developer/1) true/false, {repository} --> https://github.com/azure/osdu-devloper {branch} --> branch:main') param clusterSoftware object = { enable: true + private: false osduCore: true osduReference: true osduVersion: '' @@ -771,9 +770,9 @@ module storage 'modules/storage-account/main.bicep' = { ] // Apply Security - allowBlobPublicAccess: enableBlobPublicAccess + allowBlobPublicAccess: false publicNetworkAccess: 'Enabled' - allowSharedKeyAccess: true + allowSharedKeyAccess: false // https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/deployment-script-template?tabs=CLI#debug-deployment-scripts networkAcls: { bypass: 'AzureServices' @@ -915,7 +914,7 @@ var directoryUploads = [ ] @batchSize(1) -module gitOpsUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for item in directoryUploads: { +module gitOpsUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for item in directoryUploads: if (clusterSoftware.private) { name: '${configuration.name}-storage-${item.directory}-upload' params: { name: 'script-${storage.outputs.name}-${item.directory}' @@ -947,49 +946,6 @@ module gitOpsUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for }] -//TODO: This needs to be removed and moved into a kubernetes job. -// module manifestDagShareUpload 'modules/script-share-upload/main.bicep' = { -// name: '${configuration.name}-storage-dag-upload-manifest' -// params: { -// newStorageAccount: true -// location: location -// storageAccountName: storage.outputs.name -// identityName: stampIdentity.outputs.name - -// shareName: 'airflow-dags' -// filename: 'src/osdu_dags' -// compress: true -// fileurl: 'https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-dags/-/archive/master/ingestion-dags-master.tar.gz' -// } -// dependsOn: [ -// stampIdentity -// storage -// ] -// } - -//TODO: This needs to be removed and moved into a kubernetes job. -module csvDagShareUpload 'modules/script-share-csvdag/main.bicep' = { - name: '${configuration.name}-storage-dag-upload-csv' - params: { - newStorageAccount: true - location: location - storageAccountName: storage.outputs.name - identityName: stampIdentity.outputs.name - - shareName: 'airflow-dags' - filename: 'airflowdags' - fileurl: 'https://community.opengroup.org/osdu/platform/data-flow/ingestion/csv-parser/csv-parser/-/archive/master/csv-parser-master.tar.gz' - keyVaultUrl: keyvault.outputs.uri - insightsKey: insights.outputs.instrumentationKey - clientId: applicationClientId - clientSecret: applicationClientSecret - } - dependsOn: [ - stampIdentity - storage - ] -} - /* .______ ___ .______ .___________. __ .___________. __ ______ .__ __. | _ \ / \ | _ \ | || | | || | / __ \ | \ | | @@ -1021,7 +977,7 @@ module partitionBlade 'modules/blade_partition.bicep' = { kvName: keyvault.outputs.name natClusterIP: clusterBlade.outputs.natClusterIP - enableBlobPublicAccess: enableBlobPublicAccess + enableBlobPublicAccess: false partitions: configuration.partitions managedIdentityName: stampIdentity.outputs.name @@ -1074,6 +1030,8 @@ module configBlade 'modules/blade_configuration.bicep' = { enableExperimental: experimentalSoftware.enable == 'true' ? true : false enableAdminUI: experimentalSoftware.adminUI == 'true' ? true : false + sourceHost: clusterSoftware.private ? 'azureBlob' : 'gitRepository' + emailAddress: emailAddress applicationClientId: applicationClientId applicationClientPrincipalOid: applicationClientPrincipalOid @@ -1115,6 +1073,8 @@ module configBlade 'modules/blade_configuration.bicep' = { ] } + +// TODO: ACL breaks Partition Service Issue: https://github.com/Azure/osdu-developer/issues/230 // module storageAcl 'modules/network_acl_storage.bicep' = { // name: '${configuration.name}-storage-acl' // params: { @@ -1124,8 +1084,6 @@ module configBlade 'modules/blade_configuration.bicep' = { // natClusterIP: clusterBlade.outputs.natClusterIP // } // dependsOn: [ -// manifestDagShareUpload -// csvDagShareUpload // gitOpsUpload // ] // } diff --git a/bicep/main.parameters.json b/bicep/main.parameters.json index 5db650e1..34aac859 100644 --- a/bicep/main.parameters.json +++ b/bicep/main.parameters.json @@ -26,9 +26,6 @@ "enablePodSubnet": { "value": "${ENABLE_POD_SUBNET}" }, - "enableBlobPublicAccess": { - "value": "${ENABLE_BLOB_PUBLIC_ACCESS}" - }, "clusterConfiguration": { "value": { "enableNodeAutoProvisioning": "${ENABLE_NODE_AUTO_PROVISIONING}", @@ -63,6 +60,7 @@ "clusterSoftware": { "value": { "enable": "${ENABLE_SOFTWARE}", + "private": "${ENABLE_PRIVATE_SOFTWARE}", "osduVersion": "${SOFTWARE_VERSION}", "osduCore": "${ENABLE_OSDU_CORE}", "osduReference": "${ENABLE_OSDU_REFERENCE}", diff --git a/bicep/modules/blade_partition.bicep b/bicep/modules/blade_partition.bicep index ddb65dd4..60d75115 100644 --- a/bicep/modules/blade_partition.bicep +++ b/bicep/modules/blade_partition.bicep @@ -668,6 +668,7 @@ module partitonNamespace 'br/public:avm/res/service-bus/namespace:0.9.1' = [for }] +// TODO: This should be moved to the Kubernetes Job. module blobUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for (partition, index) in partitions: { name: '${bladeConfig.sectionName}-storage-blob-upload-${index}' params: { @@ -697,6 +698,21 @@ module blobUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for ( } }] + +// TODO: ACL can only be applied after the blob upload. +module storageAcl './network_acl_storage.bicep' = [for (partition, index) in partitions: { + name: '${bladeConfig.sectionName}-storage-acl-${index}' + params: { + storageName: storage[index].outputs.name + location: location + skuName: partitionLayerConfig.storage.sku + natClusterIP: natClusterIP + } + dependsOn: [ + blobUpload[index] + ] +}] + module partitionSecrets './keyvault_secrets_partition.bicep' = [for (partition, index) in partitions: { name: '${bladeConfig.sectionName}-secrets-${index}' params: { diff --git a/docs/src/feature_flags.md b/docs/src/feature_flags.md index 365add3c..b9afe571 100644 --- a/docs/src/feature_flags.md +++ b/docs/src/feature_flags.md @@ -50,7 +50,6 @@ Infrastructure customizations can be modified using the following feature flags. |---------------------------|-----------------------------------------------------------------------------| | CLUSTER_INGRESS | Specifies the Ingress type for the cluster (External, Internal, or Both) | | CLUSTER_VM_SIZE | Overrides the default server type with a custom VM size | -| ENABLE_BLOB_PUBLIC_ACCESS | Enables public access for storage account blob (False by default) | | ENABLE_NODE_AUTO_PROVISIONING | Enables node auto provisioning (True by default) | | ENABLE_PRIVATE_CLUSTER | Enables private cluster (False by default) | @@ -62,6 +61,7 @@ Software customizations can be modified using the following feature flags. | Feature Flag | Description | |---------------------------|-----------------------------------------------------------------------------| | ENABLE_SOFTWARE | Disables loading of all software when set to false (True by default) | +| ENABLE_PRIVATE_SOFTWARE | Software from Azure Blob instead of Git Repositories (False by default) | | ENABLE_OSDU_CORE | Disables loading of the osdu core services (True by default) | | ENABLE_OSDU_REFERENCE | Disables loading of the osdu reference services (True by default) | | SOFTWARE_VERSION | Sets the version (branch) of OSDU to be installed (release-0-27) | From 2db5ee903ddfc026643034eb9c86972db894a359 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sat, 9 Nov 2024 17:08:54 -0600 Subject: [PATCH 50/87] Fixed boolean error. --- bicep/main.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bicep/main.bicep b/bicep/main.bicep index 3e13f623..d556a541 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -914,7 +914,7 @@ var directoryUploads = [ ] @batchSize(1) -module gitOpsUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for item in directoryUploads: if (clusterSoftware.private) { +module gitOpsUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for item in directoryUploads: if (clusterSoftware.private == 'true') { name: '${configuration.name}-storage-${item.directory}-upload' params: { name: 'script-${storage.outputs.name}-${item.directory}' From bbfae301e4ce553bbb99b3af0fca82bf2b34e4e1 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sat, 9 Nov 2024 17:46:41 -0600 Subject: [PATCH 51/87] Bug fix boolean. --- bicep/main.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bicep/main.bicep b/bicep/main.bicep index d556a541..966a055e 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -1030,7 +1030,7 @@ module configBlade 'modules/blade_configuration.bicep' = { enableExperimental: experimentalSoftware.enable == 'true' ? true : false enableAdminUI: experimentalSoftware.adminUI == 'true' ? true : false - sourceHost: clusterSoftware.private ? 'azureBlob' : 'gitRepository' + sourceHost: clusterSoftware.private == 'true' ? 'azureBlob' : 'gitRepository' emailAddress: emailAddress applicationClientId: applicationClientId From 913ce26cd9a69309cf272147304c83efd5a7a121 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 10:48:56 -0600 Subject: [PATCH 52/87] Added Legal COO kubernetes Job. --- Legal_COO.json | 1472 +++++++++++++++++++++ bicep/main.bicep | 32 +- software/applications/osdu-core/base.yaml | 8 +- 3 files changed, 1497 insertions(+), 15 deletions(-) create mode 100644 Legal_COO.json diff --git a/Legal_COO.json b/Legal_COO.json new file mode 100644 index 00000000..4dfbe41f --- /dev/null +++ b/Legal_COO.json @@ -0,0 +1,1472 @@ +[{ + "name": "Andorra", + "alpha2": "AD", + "numeric": 16, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "United Arab Emirates", + "alpha2": "AE", + "numeric": 784, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Afghanistan", + "alpha2": "AF", + "numeric": 4, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Antigua and Barbuda", + "alpha2": "AG", + "numeric": 28, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Anguilla", + "alpha2": "AI", + "numeric": 660, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Albania", + "alpha2": "AL", + "numeric": 8, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Armenia", + "alpha2": "AM", + "numeric": 51, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Netherlands Antilles", + "alpha2": "AN", + "numeric": 530, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Angola", + "alpha2": "AO", + "numeric": 24, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Antarctica", + "alpha2": "AQ", + "numeric": 10, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Argentina", + "alpha2": "AR", + "numeric": 32, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "American Samoa", + "alpha2": "AS", + "numeric": 16, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Austria", + "alpha2": "AT", + "numeric": 40, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Australia", + "alpha2": "AU", + "numeric": 36, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Aruba", + "alpha2": "AW", + "numeric": 533, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Aland Islands", + "alpha2": "AX", + "numeric": 248, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Azerbaijan", + "alpha2": "AZ", + "numeric": 31, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Bosnia and Herzegovina", + "alpha2": "BA", + "numeric": 70, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Barbados", + "alpha2": "BB", + "numeric": 52, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Bangladesh", + "alpha2": "BD", + "numeric": 50, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Belgium", + "alpha2": "BE", + "numeric": 56, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Burkina Faso", + "alpha2": "BF", + "numeric": 854, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Bulgaria", + "alpha2": "BG", + "numeric": 100, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Bahrain", + "alpha2": "BH", + "numeric": 48, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Burundi", + "alpha2": "BI", + "numeric": 108, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Benin", + "alpha2": "BJ", + "numeric": 204, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Saint Barthelemy", + "alpha2": "BL", + "numeric": 652, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Bermuda", + "alpha2": "BM", + "numeric": 60, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Brunei Darussalam", + "alpha2": "BN", + "numeric": 96, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Bolivia", + "alpha2": "BO", + "numeric": 68, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Brazil", + "alpha2": "BR", + "numeric": 76, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Bahamas", + "alpha2": "BS", + "numeric": 44, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Bhutan", + "alpha2": "BT", + "numeric": 64, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Bouvet Island", + "alpha2": "BV", + "numeric": 74, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Botswana", + "alpha2": "BW", + "numeric": 72, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Belarus", + "alpha2": "BY", + "numeric": 112, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Belize", + "alpha2": "BZ", + "numeric": 84, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Canada", + "alpha2": "CA", + "numeric": 124, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Cocos Islands", + "alpha2": "CC", + "numeric": 166, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "The Democratic Republic of the Congo", + "alpha2": "CD", + "numeric": 180, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Central African Republic", + "alpha2": "CF", + "numeric": 140, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Congo", + "alpha2": "CG", + "numeric": 178, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Switzerland", + "alpha2": "CH", + "numeric": 756, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Cote d'Ivoire", + "alpha2": "CI", + "numeric": 384, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Cook Islands", + "alpha2": "CK", + "numeric": 184, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Chile", + "alpha2": "CL", + "numeric": 152, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Cameroon", + "alpha2": "CM", + "numeric": 120, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "China", + "alpha2": "CN", + "numeric": 156, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Colombia", + "alpha2": "CO", + "numeric": 170, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Costa Rica", + "alpha2": "CR", + "numeric": 188, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Cuba", + "alpha2": "CU", + "numeric": 192, + "residencyRisk": "Embargoed", + "typesNotApplyDataResidency": [] +}, { + "name": "Cape Verde", + "alpha2": "CV", + "numeric": 132, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Christmas Island", + "alpha2": "CX", + "numeric": 162, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Cyprus", + "alpha2": "CY", + "numeric": 196, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Czech Republic", + "alpha2": "CZ", + "numeric": 203, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Germany", + "alpha2": "DE", + "numeric": 276, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Djibouti", + "alpha2": "DJ", + "numeric": 262, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Denmark", + "alpha2": "DK", + "numeric": 208, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Dominica", + "alpha2": "DM", + "numeric": 212, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Dominican Republic", + "alpha2": "DO", + "numeric": 214, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Algeria", + "alpha2": "DZ", + "numeric": 12, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Ecuador", + "alpha2": "EC", + "numeric": 218, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Estonia", + "alpha2": "EE", + "numeric": 233, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Egypt", + "alpha2": "EG", + "numeric": 818, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Western Sahara", + "alpha2": "EH", + "numeric": 732, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Eritrea", + "alpha2": "ER", + "numeric": 232, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Spain", + "alpha2": "ES", + "numeric": 724, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Ethiopia", + "alpha2": "ET", + "numeric": 231, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Finland", + "alpha2": "FI", + "numeric": 246, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Fiji", + "alpha2": "FJ", + "numeric": 242, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Falkland Islands", + "alpha2": "FK", + "numeric": 238, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Federated States of Micronesia", + "alpha2": "FM", + "numeric": 583, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Faroe Islands", + "alpha2": "FO", + "numeric": 234, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "France", + "alpha2": "FR", + "numeric": 250, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Gabon", + "alpha2": "GA", + "numeric": 266, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "United Kingdom", + "alpha2": "GB", + "numeric": 826, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Grenada", + "alpha2": "GD", + "numeric": 308, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Georgia", + "alpha2": "GE", + "numeric": 268, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "French Guiana", + "alpha2": "GF", + "numeric": 254, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Guernsey", + "alpha2": "GG", + "numeric": 831, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Ghana", + "alpha2": "GH", + "numeric": 288, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Gibraltar", + "alpha2": "GI", + "numeric": 292, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Greenland", + "alpha2": "GL", + "numeric": 304, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Gambia", + "alpha2": "GM", + "numeric": 270, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Guinea", + "alpha2": "GN", + "numeric": 324, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Guadeloupe", + "alpha2": "GP", + "numeric": 312, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Equatorial Guinea", + "alpha2": "GQ", + "numeric": 226, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Greece", + "alpha2": "GR", + "numeric": 300, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "South Georgia and the South Sandwich Islands", + "alpha2": "GS", + "numeric": 239, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Guatemala", + "alpha2": "GT", + "numeric": 320, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Guam", + "alpha2": "GU", + "numeric": 316, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Guinea-Bissau", + "alpha2": "GW", + "numeric": 624, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Guyana", + "alpha2": "GY", + "numeric": 328, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Hong Kong", + "alpha2": "HK", + "numeric": 344, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Heard Island and McDonald Islands", + "alpha2": "HM", + "numeric": 334, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Honduras", + "alpha2": "HN", + "numeric": 340, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Croatia", + "alpha2": "HR", + "numeric": 191, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Haiti", + "alpha2": "HT", + "numeric": 332, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Hungary", + "alpha2": "HU", + "numeric": 348, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Indonesia", + "alpha2": "ID", + "numeric": 360, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Ireland", + "alpha2": "IE", + "numeric": 372, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Israel", + "alpha2": "IL", + "numeric": 376, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Isle of Man", + "alpha2": "IM", + "numeric": 833, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "India", + "alpha2": "IN", + "numeric": 356, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "British Indian Ocean Territory", + "alpha2": "IO", + "numeric": 86, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Iraq", + "alpha2": "IQ", + "numeric": 368, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Islamic Republic of Iran", + "alpha2": "IR", + "numeric": 364, + "residencyRisk": "Embargoed", + "typesNotApplyDataResidency": [] +}, { + "name": "Iceland", + "alpha2": "IS", + "numeric": 352, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Italy", + "alpha2": "IT", + "numeric": 380, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Jersey", + "alpha2": "JE", + "numeric": 832, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Jamaica", + "alpha2": "JM", + "numeric": 388, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Jordan", + "alpha2": "JO", + "numeric": 400, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Japan", + "alpha2": "JP", + "numeric": 392, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Kenya", + "alpha2": "KE", + "numeric": 404, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Kyrgyzstan", + "alpha2": "KG", + "numeric": 417, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Cambodia", + "alpha2": "KH", + "numeric": 116, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Kiribati", + "alpha2": "KI", + "numeric": 296, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Comoros", + "alpha2": "KM", + "numeric": 174, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Saint Kitts and Nevis", + "alpha2": "KN", + "numeric": 659, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Democratic People's Republic of Korea", + "alpha2": "KP", + "numeric": 408, + "residencyRisk": "Embargoed", + "typesNotApplyDataResidency": [] +}, { + "name": "Republic of Korea", + "alpha2": "KR", + "numeric": 410, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Kuwait", + "alpha2": "KW", + "numeric": 414, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Cayman Islands", + "alpha2": "KY", + "numeric": 136, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Kazakhstan", + "alpha2": "KZ", + "numeric": 398, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Lao People's Democratic Republic", + "alpha2": "LA", + "numeric": 418, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Lebanon", + "alpha2": "LB", + "numeric": 422, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Saint Lucia", + "alpha2": "LC", + "numeric": 662, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Liechtenstein", + "alpha2": "LI", + "numeric": 438, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Sri Lanka", + "alpha2": "LK", + "numeric": 144, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Liberia", + "alpha2": "LR", + "numeric": 430, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Lesotho", + "alpha2": "LS", + "numeric": 426, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Lithuania", + "alpha2": "LT", + "numeric": 440, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Luxembourg", + "alpha2": "LU", + "numeric": 442, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Latvia", + "alpha2": "LV", + "numeric": 428, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Libya", + "alpha2": "LY", + "numeric": 434, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Morocco", + "alpha2": "MA", + "numeric": 504, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Monaco", + "alpha2": "MC", + "numeric": 492, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Republic of Moldova", + "alpha2": "MD", + "numeric": 498, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Montenegro", + "alpha2": "ME", + "numeric": 499, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Saint Martin", + "alpha2": "MF", + "numeric": 663, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Madagascar", + "alpha2": "MG", + "numeric": 450, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Marshall Islands", + "alpha2": "MH", + "numeric": 584, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "The former Yugoslav Republic of Macedonia", + "alpha2": "MK", + "numeric": 807, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Mali", + "alpha2": "ML", + "numeric": 466, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Myanmar", + "alpha2": "MM", + "numeric": 104, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Mongolia", + "alpha2": "MN", + "numeric": 496, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Macao", + "alpha2": "MO", + "numeric": 446, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Northern Mariana Islands", + "alpha2": "MP", + "numeric": 580, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Martinique", + "alpha2": "MQ", + "numeric": 474, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Mauritania", + "alpha2": "MR", + "numeric": 478, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Montserrat", + "alpha2": "MS", + "numeric": 500, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Malta", + "alpha2": "MT", + "numeric": 470, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Mauritius", + "alpha2": "MU", + "numeric": 480, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Maldives", + "alpha2": "MV", + "numeric": 462, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Malawi", + "alpha2": "MW", + "numeric": 454, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Mexico", + "alpha2": "MX", + "numeric": 484, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Malaysia", + "alpha2": "MY", + "numeric": 458, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Mozambique", + "alpha2": "MZ", + "numeric": 508, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Namibia", + "alpha2": "NA", + "numeric": 516, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "New Caledonia", + "alpha2": "NC", + "numeric": 540, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Niger", + "alpha2": "NE", + "numeric": 562, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Norfolk Island", + "alpha2": "NF", + "numeric": 574, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Nigeria", + "alpha2": "NG", + "numeric": 566, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Nicaragua", + "alpha2": "NI", + "numeric": 558, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Netherlands", + "alpha2": "NL", + "numeric": 528, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Norway", + "alpha2": "NO", + "numeric": 578, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Nepal", + "alpha2": "NP", + "numeric": 524, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Nauru", + "alpha2": "NR", + "numeric": 520, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Niue", + "alpha2": "NU", + "numeric": 570, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "New Zealand", + "alpha2": "NZ", + "numeric": 554, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Oman", + "alpha2": "OM", + "numeric": 512, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Panama", + "alpha2": "PA", + "numeric": 591, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Peru", + "alpha2": "PE", + "numeric": 604, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "French Polynesia", + "alpha2": "PF", + "numeric": 258, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Papua New Guinea", + "alpha2": "PG", + "numeric": 598, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Philippines", + "alpha2": "PH", + "numeric": 608, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Pakistan", + "alpha2": "PK", + "numeric": 586, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Poland", + "alpha2": "PL", + "numeric": 616, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Saint Pierre and Miquelon", + "alpha2": "PM", + "numeric": 666, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Pitcairn", + "alpha2": "PN", + "numeric": 612, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Puerto Rico", + "alpha2": "PR", + "numeric": 630, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Palestinian Territory", + "alpha2": "PS", + "numeric": 275, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Portugal", + "alpha2": "PT", + "numeric": 620, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Palau", + "alpha2": "PW", + "numeric": 585, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Paraguay", + "alpha2": "PY", + "numeric": 600, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Qatar", + "alpha2": "QA", + "numeric": 634, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Reunion", + "alpha2": "RE", + "numeric": 638, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Romania", + "alpha2": "RO", + "numeric": 642, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Serbia", + "alpha2": "RS", + "numeric": 688, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Russian Federation", + "alpha2": "RU", + "numeric": 643, + "residencyRisk": "Embargoed", + "typesNotApplyDataResidency": [] +}, { + "name": "Rwanda", + "alpha2": "RW", + "numeric": 646, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Saudi Arabia", + "alpha2": "SA", + "numeric": 682, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Solomon Islands", + "alpha2": "SB", + "numeric": 90, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Seychelles", + "alpha2": "SC", + "numeric": 690, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Sudan", + "alpha2": "SD", + "numeric": 729, + "residencyRisk": "Embargoed", + "typesNotApplyDataResidency": [] +}, { + "name": "Sweden", + "alpha2": "SE", + "numeric": 752, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Singapore", + "alpha2": "SG", + "numeric": 702, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Saint Helena", + "alpha2": "SH", + "numeric": 654, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Slovenia", + "alpha2": "SI", + "numeric": 705, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Svalbard and Jan Mayen", + "alpha2": "SJ", + "numeric": 744, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Slovakia", + "alpha2": "SK", + "numeric": 703, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Sierra Leone", + "alpha2": "SL", + "numeric": 694, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "San Marino", + "alpha2": "SM", + "numeric": 674, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Senegal", + "alpha2": "SN", + "numeric": 686, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Somalia", + "alpha2": "SO", + "numeric": 706, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Suriname", + "alpha2": "SR", + "numeric": 740, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "South Sudan", + "alpha2": "SS", + "numeric": 728, + "residencyRisk": "Embargoed", + "typesNotApplyDataResidency": [] +}, { + "name": "Sao Tome and Principe", + "alpha2": "ST", + "numeric": 678, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "El Salvador", + "alpha2": "SV", + "numeric": 222, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Syrian Arab Republic", + "alpha2": "SY", + "numeric": 760, + "residencyRisk": "Embargoed", + "typesNotApplyDataResidency": [] +}, { + "name": "Swaziland", + "alpha2": "SZ", + "numeric": 748, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Turks and Caicos Islands", + "alpha2": "TC", + "numeric": 796, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Chad", + "alpha2": "TD", + "numeric": 148, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Togo", + "alpha2": "TG", + "numeric": 768, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Thailand", + "alpha2": "TH", + "numeric": 764, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Tajikistan", + "alpha2": "TJ", + "numeric": 762, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Tokelau", + "alpha2": "TK", + "numeric": 772, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Timor-Leste", + "alpha2": "TL", + "numeric": 626, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Turkmenistan", + "alpha2": "TM", + "numeric": 795, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Tunisia", + "alpha2": "TN", + "numeric": 788, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Tonga", + "alpha2": "TO", + "numeric": 776, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Turkey", + "alpha2": "TR", + "numeric": 792, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Trinidad and Tobago", + "alpha2": "TT", + "numeric": 780, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Tuvalu", + "alpha2": "TV", + "numeric": 798, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Taiwan, Province of China", + "alpha2": "TW", + "numeric": 158, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "United Republic of Tanzania", + "alpha2": "TZ", + "numeric": 834, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Ukraine", + "alpha2": "UA", + "numeric": 804, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Uganda", + "alpha2": "UG", + "numeric": 800, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "United States Minor Outlying Islands", + "alpha2": "UM", + "numeric": 581, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "United States", + "alpha2": "US", + "numeric": 840, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Uruguay", + "alpha2": "UY", + "numeric": 858, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Uzbekistan", + "alpha2": "UZ", + "numeric": 860, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Saint Vincent and the Grenadines", + "alpha2": "VC", + "numeric": 670, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Venezuela", + "alpha2": "VE", + "numeric": 862, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "British Virgin Islands", + "alpha2": "VG", + "numeric": 92, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Virgin Islands, U.S.", + "alpha2": "VI", + "numeric": 850, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Vietnam", + "alpha2": "VN", + "numeric": 704, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Vanuatu", + "alpha2": "VU", + "numeric": 548, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Wallis and Futuna", + "alpha2": "WF", + "numeric": 876, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Samoa", + "alpha2": "WS", + "numeric": 882, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Yemen", + "alpha2": "YE", + "numeric": 887, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "South Africa", + "alpha2": "ZA", + "numeric": 710, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Zambia", + "alpha2": "ZM", + "numeric": 894, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Zimbabwe", + "alpha2": "ZW", + "numeric": 716, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Default", + "alpha2": "XX", + "numeric": 999, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +} +] diff --git a/bicep/main.bicep b/bicep/main.bicep index 966a055e..27f1f903 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -772,7 +772,10 @@ module storage 'modules/storage-account/main.bicep' = { // Apply Security allowBlobPublicAccess: false publicNetworkAccess: 'Enabled' - allowSharedKeyAccess: false + + // TODO: This is required for Partition Service to access the storage account. Issue: https://github.com/Azure/osdu-developer/issues/230 + allowSharedKeyAccess: true + // https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/deployment-script-template?tabs=CLI#debug-deployment-scripts networkAcls: { bypass: 'AzureServices' @@ -945,6 +948,19 @@ module gitOpsUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for } }] +module storageAcl 'modules/network_acl_storage.bicep' = { + name: '${configuration.name}-storage-acl' + params: { + storageName: storage.outputs.name + location: location + skuName: configuration.storage.sku + natClusterIP: clusterBlade.outputs.natClusterIP + } + dependsOn: [ + gitOpsUpload + ] +} + /* .______ ___ .______ .___________. __ .___________. __ ______ .__ __. @@ -1074,19 +1090,7 @@ module configBlade 'modules/blade_configuration.bicep' = { } -// TODO: ACL breaks Partition Service Issue: https://github.com/Azure/osdu-developer/issues/230 -// module storageAcl 'modules/network_acl_storage.bicep' = { -// name: '${configuration.name}-storage-acl' -// params: { -// storageName: storage.outputs.name -// location: location -// skuName: configuration.storage.sku -// natClusterIP: clusterBlade.outputs.natClusterIP -// } -// dependsOn: [ -// gitOpsUpload -// ] -// } + // =============== // // Outputs // diff --git a/software/applications/osdu-core/base.yaml b/software/applications/osdu-core/base.yaml index 76b91eb3..81772fe5 100644 --- a/software/applications/osdu-core/base.yaml +++ b/software/applications/osdu-core/base.yaml @@ -31,4 +31,10 @@ spec: defaultCpuRequests: "0.5" defaultMemoryRequests: "1Gi" defaultCpuLimits: "2" - defaultMemoryLimits: "4Gi" \ No newline at end of file + defaultMemoryLimits: "4Gi" + share: + enabled: true + items: + - name: legal + file: "Legal_COO.json" + url: "https://raw.githubusercontent.com/Azure/osdu-developer/refs/heads/main/bicep/modules/script-blob-upload/Legal_COO.json" From 26806027988bc6974a5fd66c49ed5908781d657f Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 13:26:34 -0600 Subject: [PATCH 53/87] Added Legal COO kubernetes Job. --- .../templates/storage-container-job.yaml | 77 +++++++++++++++++++ ...{share-job.yaml => storage-share-job.yaml} | 0 software/applications/osdu-core/base.yaml | 3 +- 3 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 charts/osdu-developer-base/templates/storage-container-job.yaml rename charts/osdu-developer-base/templates/{share-job.yaml => storage-share-job.yaml} (100%) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml new file mode 100644 index 00000000..fc0c5e85 --- /dev/null +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -0,0 +1,77 @@ +{{- if .Values.blobUpload.enabled -}} +{{- $configMap := (lookup "v1" "ConfigMap" .Release.Namespace "configmap-services").data }} +{{- $storageAccounts := list }} +{{- range $key, $value := $configMap }} +{{- if hasPrefix "partition_storage_name_" $key }} +{{- $storageAccounts = append $storageAccounts $value }} +{{- end }} +{{- end }} + +{{- if $storageAccounts }} +{{- range $storageAccount := $storageAccounts }} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ $.Release.Name }}-blob-storage-{{ $storageAccount }} +provisioner: blob.csi.azure.com +parameters: + storageAccount: {{ $storageAccount }} + containerName: {{ $.Values.blobUpload.container }} + clientID: {{ $.Values.blobUpload.clientId | default $configMap.azure_msi_client_id }} +mountOptions: + - -o allow_other + - --file-cache-timeout-in-seconds=120 +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ $.Release.Name }}-blob-pvc-{{ $storageAccount }} + namespace: {{ $.Release.Namespace }} +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 5Gi + storageClassName: {{ $.Release.Name }}-blob-storage-{{ $storageAccount }} +--- +{{- range $.Values.blobUpload.items }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $.Release.Name }}-blob-upload-{{ .name }}-{{ $storageAccount }} + namespace: {{ $.Release.Namespace }} +spec: + ttlSecondsAfterFinished: 300 + template: + spec: + serviceAccountName: workload-identity-sa + volumes: + - name: blob-storage + persistentVolumeClaim: + claimName: {{ $.Release.Name }}-blob-pvc-{{ $storageAccount }} + containers: + - name: blob-upload + image: mcr.microsoft.com/cbl-mariner/base/core:2.0 + command: ["/bin/sh"] + args: + - -c + - | + # Install curl + tdnf install -y curl + + # Download the file + echo "Downloading file from {{ .url }}" + curl -kso {{ .file }} "{{ .url }}" + + # Copy to mounted blob container + cp {{ .file }} /mnt/blob/{{ $.Values.blobUpload.container }}/{{ .file }} + + echo "File uploaded to container {{ $.Values.blobUpload.container }} in storage account {{ $storageAccount }}" + volumeMounts: + - name: blob-storage + mountPath: /mnt/blob + restartPolicy: Never +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/osdu-developer-base/templates/share-job.yaml b/charts/osdu-developer-base/templates/storage-share-job.yaml similarity index 100% rename from charts/osdu-developer-base/templates/share-job.yaml rename to charts/osdu-developer-base/templates/storage-share-job.yaml diff --git a/software/applications/osdu-core/base.yaml b/software/applications/osdu-core/base.yaml index 81772fe5..454bd9c2 100644 --- a/software/applications/osdu-core/base.yaml +++ b/software/applications/osdu-core/base.yaml @@ -32,8 +32,9 @@ spec: defaultMemoryRequests: "1Gi" defaultCpuLimits: "2" defaultMemoryLimits: "4Gi" - share: + blobUpload: enabled: true + container: "legal-service-azure-configuration" items: - name: legal file: "Legal_COO.json" From acb316c6a10e3614b5a2ec88943c2871956d552b Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 13:35:37 -0600 Subject: [PATCH 54/87] Fixing helm chart. --- charts/osdu-developer-base/templates/storage-container-job.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index fc0c5e85..e3d0f12b 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -74,4 +74,5 @@ spec: restartPolicy: Never {{- end }} {{- end }} +{{- end }} {{- end }} \ No newline at end of file From 5a660a4fc9309f1dfacc8778506b40e151777457 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 13:59:37 -0600 Subject: [PATCH 55/87] Fixing helm chart. --- .../osdu-developer-base/templates/storage-container-job.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index e3d0f12b..cab87e0e 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -1,5 +1,5 @@ {{- if .Values.blobUpload.enabled -}} -{{- $configMap := (lookup "v1" "ConfigMap" .Release.Namespace "configmap-services").data }} +{{- $configMap := (lookup "v1" "ConfigMap" "osdu-core" "configmap-services").data }} {{- $storageAccounts := list }} {{- range $key, $value := $configMap }} {{- if hasPrefix "partition_storage_name_" $key }} @@ -17,7 +17,7 @@ provisioner: blob.csi.azure.com parameters: storageAccount: {{ $storageAccount }} containerName: {{ $.Values.blobUpload.container }} - clientID: {{ $.Values.blobUpload.clientId | default $configMap.azure_msi_client_id }} + clientID: {{ $.Values.blobUpload.clientId | default $.Values.azure.clientId }} mountOptions: - -o allow_other - --file-cache-timeout-in-seconds=120 From dad72c5faadf4f5cfedef2ce7ce0974e8fcd604f Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 14:16:02 -0600 Subject: [PATCH 56/87] Fixing helm chart. --- .../osdu-developer-base/templates/storage-container-job.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index cab87e0e..68f61ecb 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -7,7 +7,7 @@ {{- end }} {{- end }} -{{- if $storageAccounts }} +# {{- if $storageAccounts }} {{- range $storageAccount := $storageAccounts }} apiVersion: storage.k8s.io/v1 kind: StorageClass @@ -75,4 +75,4 @@ spec: {{- end }} {{- end }} {{- end }} -{{- end }} \ No newline at end of file +# {{- end }} \ No newline at end of file From 84df91619f5812dc2c7996e15ec82258bdc9c50d Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 14:33:53 -0600 Subject: [PATCH 57/87] Updated the chart. --- .../templates/storage-container-job.yaml | 23 ++++++++++++------- charts/osdu-developer-base/values.yaml | 9 ++++---- software/applications/osdu-core/base.yaml | 3 ++- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index 68f61ecb..e331b65f 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -1,13 +1,20 @@ -{{- if .Values.blobUpload.enabled -}} -{{- $configMap := (lookup "v1" "ConfigMap" "osdu-core" "configmap-services").data }} +{{- if (default false .Values.blobUpload.enabled) -}} {{- $storageAccounts := list }} -{{- range $key, $value := $configMap }} -{{- if hasPrefix "partition_storage_name_" $key }} -{{- $storageAccounts = append $storageAccounts $value }} -{{- end }} + +{{/* Try to get storage accounts from ConfigMap if it exists */}} +{{- if (lookup "v1" "ConfigMap" .Release.Namespace "configmap-services") }} + {{- $configMap := (lookup "v1" "ConfigMap" .Release.Namespace "configmap-services").data }} + {{- range $key, $value := $configMap }} + {{- if hasPrefix "partition_storage_name_" $key }} + {{- $storageAccounts = append $storageAccounts $value }} + {{- end }} + {{- end }} +{{/* Otherwise use storage accounts from values if provided */}} +{{- else if .Values.blobUpload.storageAccounts }} + {{- $storageAccounts = .Values.blobUpload.storageAccounts }} {{- end }} -# {{- if $storageAccounts }} +{{- if $storageAccounts }} {{- range $storageAccount := $storageAccounts }} apiVersion: storage.k8s.io/v1 kind: StorageClass @@ -75,4 +82,4 @@ spec: {{- end }} {{- end }} {{- end }} -# {{- end }} \ No newline at end of file +{{- end }} \ No newline at end of file diff --git a/charts/osdu-developer-base/values.yaml b/charts/osdu-developer-base/values.yaml index 5c7c0e50..8c9b14c3 100644 --- a/charts/osdu-developer-base/values.yaml +++ b/charts/osdu-developer-base/values.yaml @@ -36,6 +36,10 @@ resourceLimits: defaultCpuLimits: "1" defaultMemoryLimits: "4Gi" +blobUpload: + enabled: false + container: "legal-service-azure-configuration" + share: enabled: false items: @@ -47,8 +51,3 @@ share: file: "folder_in_archive" compress: true -airflow: - manifestdag: - enabled: false - csvdag: - enabled: false diff --git a/software/applications/osdu-core/base.yaml b/software/applications/osdu-core/base.yaml index 454bd9c2..a22fd44f 100644 --- a/software/applications/osdu-core/base.yaml +++ b/software/applications/osdu-core/base.yaml @@ -34,7 +34,8 @@ spec: defaultMemoryLimits: "4Gi" blobUpload: enabled: true - container: "legal-service-azure-configuration" + storageAccounts: + - "partition_storage_name_0" items: - name: legal file: "Legal_COO.json" From 40dace2f3622eb1474c7adc2c529a31151f3684d Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 14:41:43 -0600 Subject: [PATCH 58/87] Updated the chart. --- .../templates/storage-container-job.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index e331b65f..6b40126e 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -9,17 +9,16 @@ {{- $storageAccounts = append $storageAccounts $value }} {{- end }} {{- end }} -{{/* Otherwise use storage accounts from values if provided */}} {{- else if .Values.blobUpload.storageAccounts }} {{- $storageAccounts = .Values.blobUpload.storageAccounts }} {{- end }} {{- if $storageAccounts }} -{{- range $storageAccount := $storageAccounts }} +{{- range $i, $storageAccount := $storageAccounts }} apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: {{ $.Release.Name }}-blob-storage-{{ $storageAccount }} + name: {{ $.Release.Name }}-blob-storage-{{ $i }} provisioner: blob.csi.azure.com parameters: storageAccount: {{ $storageAccount }} @@ -32,7 +31,7 @@ mountOptions: apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: {{ $.Release.Name }}-blob-pvc-{{ $storageAccount }} + name: {{ $.Release.Name }}-blob-pvc-{{ $i }} namespace: {{ $.Release.Namespace }} spec: accessModes: @@ -40,13 +39,13 @@ spec: resources: requests: storage: 5Gi - storageClassName: {{ $.Release.Name }}-blob-storage-{{ $storageAccount }} + storageClassName: {{ $.Release.Name }}-blob-storage-{{ $i }} --- {{- range $.Values.blobUpload.items }} apiVersion: batch/v1 kind: Job metadata: - name: {{ $.Release.Name }}-blob-upload-{{ .name }}-{{ $storageAccount }} + name: {{ $.Release.Name }}-blob-upload-{{ .name }}-{{ $i }} namespace: {{ $.Release.Namespace }} spec: ttlSecondsAfterFinished: 300 @@ -56,7 +55,7 @@ spec: volumes: - name: blob-storage persistentVolumeClaim: - claimName: {{ $.Release.Name }}-blob-pvc-{{ $storageAccount }} + claimName: {{ $.Release.Name }}-blob-pvc-{{ $i }} containers: - name: blob-upload image: mcr.microsoft.com/cbl-mariner/base/core:2.0 From f1f833c00e41e7d0230a9865cbb9c05df06e3fbc Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 14:50:49 -0600 Subject: [PATCH 59/87] Updated the chart. --- .../osdu-developer-base/templates/storage-container-job.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index 6b40126e..ec17821d 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -23,7 +23,7 @@ provisioner: blob.csi.azure.com parameters: storageAccount: {{ $storageAccount }} containerName: {{ $.Values.blobUpload.container }} - clientID: {{ $.Values.blobUpload.clientId | default $.Values.azure.clientId }} + clientId: {{ $.Values.blobUpload.clientId | default $.Values.azure.clientId }} mountOptions: - -o allow_other - --file-cache-timeout-in-seconds=120 @@ -80,5 +80,4 @@ spec: restartPolicy: Never {{- end }} {{- end }} -{{- end }} {{- end }} \ No newline at end of file From 36364f5d553d75744b71eaa022411d2beb91bbb2 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 14:58:26 -0600 Subject: [PATCH 60/87] Updated the chart. --- charts/osdu-developer-base/templates/storage-container-job.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index ec17821d..4af2e17d 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -80,4 +80,5 @@ spec: restartPolicy: Never {{- end }} {{- end }} +{{- end }} {{- end }} \ No newline at end of file From a98af3f7f2a98ba8b1d5bd2ef7a83b13adf6bde1 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 15:29:18 -0600 Subject: [PATCH 61/87] Updated the chart. --- .../templates/storage-container-job.yaml | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index 4af2e17d..71fa606e 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -15,18 +15,28 @@ {{- if $storageAccounts }} {{- range $i, $storageAccount := $storageAccounts }} -apiVersion: storage.k8s.io/v1 -kind: StorageClass +apiVersion: v1 +kind: PersistentVolume metadata: - name: {{ $.Release.Name }}-blob-storage-{{ $i }} -provisioner: blob.csi.azure.com -parameters: - storageAccount: {{ $storageAccount }} - containerName: {{ $.Values.blobUpload.container }} - clientId: {{ $.Values.blobUpload.clientId | default $.Values.azure.clientId }} -mountOptions: - - -o allow_other - - --file-cache-timeout-in-seconds=120 + annotations: + pv.kubernetes.io/provisioned-by: blob.csi.azure.com + name: {{ $.Release.Name }}-blob-pv-{{ $i }} +spec: + capacity: + storage: 5Gi + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + mountOptions: + - -o allow_other + - --file-cache-timeout-in-seconds=120 + csi: + driver: blob.csi.azure.com + volumeHandle: {{ $storageAccount }}_{{ $.Values.blobUpload.container }} + volumeAttributes: + storageaccount: {{ $storageAccount }} + containerName: {{ $.Values.blobUpload.container }} + clientID: {{ $.Values.blobUpload.clientId | default $.Values.azure.clientId }} --- apiVersion: v1 kind: PersistentVolumeClaim @@ -39,7 +49,7 @@ spec: resources: requests: storage: 5Gi - storageClassName: {{ $.Release.Name }}-blob-storage-{{ $i }} + volumeName: {{ $.Release.Name }}-blob-pv-{{ $i }} --- {{- range $.Values.blobUpload.items }} apiVersion: batch/v1 From b4a4853ce87804c9ea100443823d220cd121c776 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 16:44:08 -0600 Subject: [PATCH 62/87] Updated the chart. --- charts/osdu-developer-base/templates/storage-container-job.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index 71fa606e..ab609421 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -44,6 +44,7 @@ metadata: name: {{ $.Release.Name }}-blob-pvc-{{ $i }} namespace: {{ $.Release.Namespace }} spec: + storageClassName: azureblob-fuse-premium accessModes: - ReadWriteMany resources: From ced3bb031e992eaca05d1e574d8d046c1f6c1acd Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 16:58:20 -0600 Subject: [PATCH 63/87] Updated the chart. --- charts/osdu-developer-base/templates/storage-container-job.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index ab609421..35bd55ab 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -22,6 +22,7 @@ metadata: pv.kubernetes.io/provisioned-by: blob.csi.azure.com name: {{ $.Release.Name }}-blob-pv-{{ $i }} spec: + storageClassName: azureblob-fuse-premium capacity: storage: 5Gi accessModes: From 013c98042886ae3eda2c6023eaca9f376dd6596d Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 17:16:18 -0600 Subject: [PATCH 64/87] Updated the resource gruop --- bicep/modules/blade_configuration.bicep | 6 ++++++ .../templates/storage-container-job.yaml | 1 + 2 files changed, 7 insertions(+) diff --git a/bicep/modules/blade_configuration.bicep b/bicep/modules/blade_configuration.bicep index 0b17c286..df6e26fe 100644 --- a/bicep/modules/blade_configuration.bicep +++ b/bicep/modules/blade_configuration.bicep @@ -288,6 +288,12 @@ var settings = [ contentType: 'text/plain' label: 'configmap-services' } + { + name: 'resource_group' + value: resourceGroup().name + contentType: 'text/plain' + label: 'configmap-services' + } { name: 'azure_msi_client_id' value: appIdentity.properties.clientId diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index 35bd55ab..63a721b2 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -38,6 +38,7 @@ spec: storageaccount: {{ $storageAccount }} containerName: {{ $.Values.blobUpload.container }} clientID: {{ $.Values.blobUpload.clientId | default $.Values.azure.clientId }} + resourceGroup: {{ (lookup "v1" "ConfigMap" $.Release.Namespace "configmap-services").data.resource_group }} --- apiVersion: v1 kind: PersistentVolumeClaim From 9e1204bd15fb1d79dccf113d88937c02a6961dcb Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 18:09:47 -0600 Subject: [PATCH 65/87] Updated the resource gruop --- bicep/modules/blade_partition.bicep | 76 +++++++++---------- .../templates/storage-container-job.yaml | 3 +- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/bicep/modules/blade_partition.bicep b/bicep/modules/blade_partition.bicep index 60d75115..b02b8944 100644 --- a/bicep/modules/blade_partition.bicep +++ b/bicep/modules/blade_partition.bicep @@ -669,49 +669,49 @@ module partitonNamespace 'br/public:avm/res/service-bus/namespace:0.9.1' = [for // TODO: This should be moved to the Kubernetes Job. -module blobUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for (partition, index) in partitions: { - name: '${bladeConfig.sectionName}-storage-blob-upload-${index}' - params: { - name: 'script-${storage[index].outputs.name}-Legal_COO' - location: location - cleanupPreference: 'Always' - retentionInterval: 'PT1H' - timeout: 'PT30M' - runOnce: true +// module blobUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for (partition, index) in partitions: { +// name: '${bladeConfig.sectionName}-storage-blob-upload-${index}' +// params: { +// name: 'script-${storage[index].outputs.name}-Legal_COO' +// location: location +// cleanupPreference: 'Always' +// retentionInterval: 'PT1H' +// timeout: 'PT30M' +// runOnce: true - managedIdentities: { - userAssignedResourcesIds: [ - stampIdentity.id - ] - } - - kind: 'AzureCLI' - azCliVersion: '2.63.0' +// managedIdentities: { +// userAssignedResourcesIds: [ +// stampIdentity.id +// ] +// } + +// kind: 'AzureCLI' +// azCliVersion: '2.63.0' - environmentVariables: [ - { name: 'CONTENT', value: loadTextContent('./deploy-scripts/Legal_COO.json') } - { name: 'FILE_NAME', value: 'Legal_COO.json' } - { name: 'CONTAINER', value: 'legal-service-azure-configuration' } - { name: 'AZURE_STORAGE_ACCOUNT', value: storage[index].outputs.name } - ] - scriptContent: loadTextContent('./deploy-scripts/blob_upload.sh') - } -}] +// environmentVariables: [ +// { name: 'CONTENT', value: loadTextContent('./deploy-scripts/Legal_COO.json') } +// { name: 'FILE_NAME', value: 'Legal_COO.json' } +// { name: 'CONTAINER', value: 'legal-service-azure-configuration' } +// { name: 'AZURE_STORAGE_ACCOUNT', value: storage[index].outputs.name } +// ] +// scriptContent: loadTextContent('./deploy-scripts/blob_upload.sh') +// } +// }] // TODO: ACL can only be applied after the blob upload. -module storageAcl './network_acl_storage.bicep' = [for (partition, index) in partitions: { - name: '${bladeConfig.sectionName}-storage-acl-${index}' - params: { - storageName: storage[index].outputs.name - location: location - skuName: partitionLayerConfig.storage.sku - natClusterIP: natClusterIP - } - dependsOn: [ - blobUpload[index] - ] -}] +// module storageAcl './network_acl_storage.bicep' = [for (partition, index) in partitions: { +// name: '${bladeConfig.sectionName}-storage-acl-${index}' +// params: { +// storageName: storage[index].outputs.name +// location: location +// skuName: partitionLayerConfig.storage.sku +// natClusterIP: natClusterIP +// } +// dependsOn: [ +// blobUpload[index] +// ] +// }] module partitionSecrets './keyvault_secrets_partition.bicep' = [for (partition, index) in partitions: { name: '${bladeConfig.sectionName}-secrets-${index}' diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index 63a721b2..661a207a 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -38,7 +38,7 @@ spec: storageaccount: {{ $storageAccount }} containerName: {{ $.Values.blobUpload.container }} clientID: {{ $.Values.blobUpload.clientId | default $.Values.azure.clientId }} - resourceGroup: {{ (lookup "v1" "ConfigMap" $.Release.Namespace "configmap-services").data.resource_group }} + resourceGroup: {{ $.Values.azure.resourceGroup }} --- apiVersion: v1 kind: PersistentVolumeClaim @@ -93,5 +93,4 @@ spec: restartPolicy: Never {{- end }} {{- end }} -{{- end }} {{- end }} \ No newline at end of file From 73e99ba72f38ebd95f249b91167015f59b147663 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 18:15:07 -0600 Subject: [PATCH 66/87] Updated the chart --- .../templates/storage-container-job.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index 661a207a..236df97a 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -91,6 +91,7 @@ spec: - name: blob-storage mountPath: /mnt/blob restartPolicy: Never -{{- end }} -{{- end }} -{{- end }} \ No newline at end of file +{{- end }}{{/* end range $.Values.blobUpload.items */}} +{{- end }}{{/* end range $storageAccounts */}} +{{- end }}{{/* end if $storageAccounts */}} +{{- end }}{{/* end if .Values.blobUpload.enabled */}} \ No newline at end of file From 0f48d4a7817bee698006d10d3672046fe44b077d Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 18:31:17 -0600 Subject: [PATCH 67/87] Moved resource group to first configmap. --- bicep/modules/blade_configuration.bicep | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/bicep/modules/blade_configuration.bicep b/bicep/modules/blade_configuration.bicep index df6e26fe..3b0feba8 100644 --- a/bicep/modules/blade_configuration.bicep +++ b/bicep/modules/blade_configuration.bicep @@ -288,12 +288,6 @@ var settings = [ contentType: 'text/plain' label: 'configmap-services' } - { - name: 'resource_group' - value: resourceGroup().name - contentType: 'text/plain' - label: 'configmap-services' - } { name: 'azure_msi_client_id' value: appIdentity.properties.clientId @@ -393,11 +387,12 @@ values.yaml: | appInsightsKey: {5} appId: {6} appOid: {7} + resourceGroup: {8} ingress: internalGateway: - enabled: {8} - externalGateway: enabled: {9} + externalGateway: + enabled: {10} ''' } @@ -433,6 +428,7 @@ module appConfigMap './aks-config-map/main.bicep' = { appInsightsKey, applicationClientId, applicationClientPrincipalOid, + resourceGroup().name, clusterIngress == 'Internal' || clusterIngress == 'Both' ? 'true' : 'false', clusterIngress == 'External' || clusterIngress == 'Both' ? 'true' : 'false') ] From 803f3d0a3e7e609c6a507f9322536161e0b05712 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 19:28:53 -0600 Subject: [PATCH 68/87] Moved resource group to first configmap. --- .../templates/storage-container-job.yaml | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index 236df97a..3b3828ab 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -1,18 +1,18 @@ {{- if (default false .Values.blobUpload.enabled) -}} +{{/* Get storage accounts from ConfigMap */}} +{{- $configMap := (lookup "v1" "ConfigMap" .Release.Namespace "configmap-services").data }} {{- $storageAccounts := list }} - -{{/* Try to get storage accounts from ConfigMap if it exists */}} -{{- if (lookup "v1" "ConfigMap" .Release.Namespace "configmap-services") }} - {{- $configMap := (lookup "v1" "ConfigMap" .Release.Namespace "configmap-services").data }} - {{- range $key, $value := $configMap }} - {{- if hasPrefix "partition_storage_name_" $key }} - {{- $storageAccounts = append $storageAccounts $value }} - {{- end }} +{{- range $key, $value := $configMap }} + {{- if hasPrefix "partition_storage_name_" $key }} + {{- $storageAccounts = append $storageAccounts $value }} {{- end }} -{{- else if .Values.blobUpload.storageAccounts }} - {{- $storageAccounts = .Values.blobUpload.storageAccounts }} {{- end }} +{{/* Get resource group from config-map-values */}} +{{- $configMapData := (lookup "v1" "ConfigMap" .Release.Namespace "config-map-values").data }} +{{- $valuesYaml := fromYaml $configMapData.values\.yaml }} +{{- $resourceGroup := $valuesYaml.azure.resourceGroup }} + {{- if $storageAccounts }} {{- range $i, $storageAccount := $storageAccounts }} apiVersion: v1 @@ -38,7 +38,7 @@ spec: storageaccount: {{ $storageAccount }} containerName: {{ $.Values.blobUpload.container }} clientID: {{ $.Values.blobUpload.clientId | default $.Values.azure.clientId }} - resourceGroup: {{ $.Values.azure.resourceGroup }} + resourceGroup: {{ $resourceGroup }} --- apiVersion: v1 kind: PersistentVolumeClaim From 79412bd6d8960f4362ff7aee02eca0f3351a6f64 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 19:33:52 -0600 Subject: [PATCH 69/87] Moved resource group to first configmap. --- charts/osdu-developer-base/templates/storage-container-job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index 3b3828ab..bf7245cd 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -10,7 +10,7 @@ {{/* Get resource group from config-map-values */}} {{- $configMapData := (lookup "v1" "ConfigMap" .Release.Namespace "config-map-values").data }} -{{- $valuesYaml := fromYaml $configMapData.values\.yaml }} +{{- $valuesYaml := fromYaml $configMapData.values.yaml }} {{- $resourceGroup := $valuesYaml.azure.resourceGroup }} {{- if $storageAccounts }} From 94652070ca92ecbfdc14ff48b3f0395906eeded8 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 19:42:32 -0600 Subject: [PATCH 70/87] Updated --- charts/osdu-developer-base/templates/storage-container-job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index bf7245cd..7d6c7fc8 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -10,7 +10,7 @@ {{/* Get resource group from config-map-values */}} {{- $configMapData := (lookup "v1" "ConfigMap" .Release.Namespace "config-map-values").data }} -{{- $valuesYaml := fromYaml $configMapData.values.yaml }} +{{- $valuesYaml := fromYaml ($configMapData.values) }} {{- $resourceGroup := $valuesYaml.azure.resourceGroup }} {{- if $storageAccounts }} From c317fd8734c77a623471c47c752a8e804d72be2e Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 19:54:14 -0600 Subject: [PATCH 71/87] Updated --- .../templates/storage-container-job.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index 7d6c7fc8..ff4fb9ac 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -9,9 +9,12 @@ {{- end }} {{/* Get resource group from config-map-values */}} -{{- $configMapData := (lookup "v1" "ConfigMap" .Release.Namespace "config-map-values").data }} -{{- $valuesYaml := fromYaml ($configMapData.values) }} -{{- $resourceGroup := $valuesYaml.azure.resourceGroup }} +{{- $configMapValues := (lookup "v1" "ConfigMap" .Release.Namespace "config-map-values") }} +{{- $resourceGroup := .Values.azure.resourceGroup }} +{{- if $configMapValues }} + {{- $valuesYaml := fromYaml $configMapValues.data.values.yaml }} + {{- $resourceGroup = $valuesYaml.azure.resourceGroup }} +{{- end }} {{- if $storageAccounts }} {{- range $i, $storageAccount := $storageAccounts }} From 326f0be8369fc6725541708ee11deb0cab16dec3 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 19:55:28 -0600 Subject: [PATCH 72/87] Updated --- charts/osdu-developer-base/templates/storage-container-job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index ff4fb9ac..539e1c27 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -9,7 +9,7 @@ {{- end }} {{/* Get resource group from config-map-values */}} -{{- $configMapValues := (lookup "v1" "ConfigMap" .Release.Namespace "config-map-values") }} +{{- $configMapValues := (lookup "v1" "ConfigMap" "default" "config-map-values") }} {{- $resourceGroup := .Values.azure.resourceGroup }} {{- if $configMapValues }} {{- $valuesYaml := fromYaml $configMapValues.data.values.yaml }} From 6bcd31994a70d393e18d76ddd6f9ddeaa6ce7480 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 20:35:40 -0600 Subject: [PATCH 73/87] Updated --- .../osdu-developer-base/templates/storage-container-job.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index 539e1c27..9444babb 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -12,7 +12,8 @@ {{- $configMapValues := (lookup "v1" "ConfigMap" "default" "config-map-values") }} {{- $resourceGroup := .Values.azure.resourceGroup }} {{- if $configMapValues }} - {{- $valuesYaml := fromYaml $configMapValues.data.values.yaml }} + {{- $valuesContent := index $configMapValues.data "values.yaml" }} + {{- $valuesYaml := fromYaml $valuesContent }} {{- $resourceGroup = $valuesYaml.azure.resourceGroup }} {{- end }} From 7eac566aa63c88d3dbf404135ec083faeef2412c Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 21:01:18 -0600 Subject: [PATCH 74/87] Updated --- .../osdu-developer-base/templates/storage-container-job.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index 9444babb..368fdb7a 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -1,6 +1,7 @@ {{- if (default false .Values.blobUpload.enabled) -}} {{/* Get storage accounts from ConfigMap */}} -{{- $configMap := (lookup "v1" "ConfigMap" .Release.Namespace "configmap-services").data }} +{{- $targetNamespace := toString .Values.targetNamespace }} +{{- $configMap := (lookup "v1" "ConfigMap" $targetNamespace "configmap-services").data }} {{- $storageAccounts := list }} {{- range $key, $value := $configMap }} {{- if hasPrefix "partition_storage_name_" $key }} From 6ac9177e0ce767a9443cce6e1112aa084d8ef2bb Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 10 Nov 2024 21:30:44 -0600 Subject: [PATCH 75/87] Updated --- .../templates/storage-container-job.yaml | 67 +++++++++++++------ 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index 368fdb7a..f622a602 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -1,25 +1,52 @@ -{{- if (default false .Values.blobUpload.enabled) -}} -{{/* Get storage accounts from ConfigMap */}} -{{- $targetNamespace := toString .Values.targetNamespace }} -{{- $configMap := (lookup "v1" "ConfigMap" $targetNamespace "configmap-services").data }} +{{- if .Values.blobUpload.enabled }} +{{- $isLocal := .Values.isLocal | default false }} + +{{/* Get storage accounts from ConfigMap or use provided values when running locally */}} +{{- $targetNamespace := .Values.targetNamespace | default .Release.Namespace }} {{- $storageAccounts := list }} -{{- range $key, $value := $configMap }} - {{- if hasPrefix "partition_storage_name_" $key }} - {{- $storageAccounts = append $storageAccounts $value }} + +{{- if not $isLocal }} + {{- $configMapObj := lookup "v1" "ConfigMap" $targetNamespace "configmap-services" }} + {{- if $configMapObj }} + {{- $configMap := index $configMapObj.Object "data" }} + {{- if $configMap }} + {{- range $key, $value := $configMap }} + {{- if hasPrefix "partition_storage_name_" $key }} + {{- $storageAccounts = append $storageAccounts $value }} + {{- end }} + {{- end }} + {{- else }} + {{- fail (printf "ConfigMap 'configmap-services' in namespace '%s' has no data." $targetNamespace) }} + {{- end }} + {{- else }} + {{- fail (printf "ConfigMap 'configmap-services' not found in namespace '%s'." $targetNamespace) }} + {{- end }} +{{- else }} +{{/* Use storage accounts from values for local testing */}} + {{- $storageAccounts = .Values.blobUpload.storageAccounts | default list }} + {{- if eq (len $storageAccounts) 0 }} + {{- fail "No storage accounts provided for local rendering. Please set 'blobUpload.storageAccounts' in your values file." }} {{- end }} {{- end }} -{{/* Get resource group from config-map-values */}} -{{- $configMapValues := (lookup "v1" "ConfigMap" "default" "config-map-values") }} -{{- $resourceGroup := .Values.azure.resourceGroup }} -{{- if $configMapValues }} - {{- $valuesContent := index $configMapValues.data "values.yaml" }} - {{- $valuesYaml := fromYaml $valuesContent }} - {{- $resourceGroup = $valuesYaml.azure.resourceGroup }} +{{/* Get resource group from config-map-values or use provided value when running locally */}} +{{- $resourceGroup := .Values.azure.resourceGroup | default "default-resource-group" }} + +{{- if not $isLocal }} + {{- $configMapValuesObj := lookup "v1" "ConfigMap" "default" "config-map-values" }} + {{- if $configMapValuesObj }} + {{- $configMapValues := index $configMapValuesObj.Object "data" }} + {{- if $configMapValues }} + {{- $valuesContent := index $configMapValues "values.yaml" }} + {{- $valuesYaml := fromYaml $valuesContent }} + {{- $resourceGroup = $valuesYaml.azure.resourceGroup | default $resourceGroup }} + {{- end }} + {{- end }} {{- end }} {{- if $storageAccounts }} -{{- range $i, $storageAccount := $storageAccounts }} + {{- range $i, $storageAccount := $storageAccounts }} +--- apiVersion: v1 kind: PersistentVolume metadata: @@ -59,7 +86,7 @@ spec: storage: 5Gi volumeName: {{ $.Release.Name }}-blob-pv-{{ $i }} --- -{{- range $.Values.blobUpload.items }} + {{- range $.Values.blobUpload.items }} apiVersion: batch/v1 kind: Job metadata: @@ -96,7 +123,7 @@ spec: - name: blob-storage mountPath: /mnt/blob restartPolicy: Never -{{- end }}{{/* end range $.Values.blobUpload.items */}} -{{- end }}{{/* end range $storageAccounts */}} -{{- end }}{{/* end if $storageAccounts */}} -{{- end }}{{/* end if .Values.blobUpload.enabled */}} \ No newline at end of file + {{- end }} {{/* end range $.Values.blobUpload.items */}} + {{- end }} {{/* end range $storageAccounts */}} +{{- end }} {{/* end if $storageAccounts */}} +{{- end }} {{/* end if .Values.blobUpload.enabled */}} \ No newline at end of file From 1286f41efe0522dd1976e566f355d514b3e9b2f7 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 11 Nov 2024 07:57:30 -0600 Subject: [PATCH 76/87] Updated --- .../templates/storage-container-job.yaml | 64 +++++-------------- 1 file changed, 16 insertions(+), 48 deletions(-) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index f622a602..236df97a 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -1,52 +1,20 @@ -{{- if .Values.blobUpload.enabled }} -{{- $isLocal := .Values.isLocal | default false }} - -{{/* Get storage accounts from ConfigMap or use provided values when running locally */}} -{{- $targetNamespace := .Values.targetNamespace | default .Release.Namespace }} +{{- if (default false .Values.blobUpload.enabled) -}} {{- $storageAccounts := list }} -{{- if not $isLocal }} - {{- $configMapObj := lookup "v1" "ConfigMap" $targetNamespace "configmap-services" }} - {{- if $configMapObj }} - {{- $configMap := index $configMapObj.Object "data" }} - {{- if $configMap }} - {{- range $key, $value := $configMap }} - {{- if hasPrefix "partition_storage_name_" $key }} - {{- $storageAccounts = append $storageAccounts $value }} - {{- end }} - {{- end }} - {{- else }} - {{- fail (printf "ConfigMap 'configmap-services' in namespace '%s' has no data." $targetNamespace) }} - {{- end }} - {{- else }} - {{- fail (printf "ConfigMap 'configmap-services' not found in namespace '%s'." $targetNamespace) }} - {{- end }} -{{- else }} -{{/* Use storage accounts from values for local testing */}} - {{- $storageAccounts = .Values.blobUpload.storageAccounts | default list }} - {{- if eq (len $storageAccounts) 0 }} - {{- fail "No storage accounts provided for local rendering. Please set 'blobUpload.storageAccounts' in your values file." }} - {{- end }} -{{- end }} - -{{/* Get resource group from config-map-values or use provided value when running locally */}} -{{- $resourceGroup := .Values.azure.resourceGroup | default "default-resource-group" }} - -{{- if not $isLocal }} - {{- $configMapValuesObj := lookup "v1" "ConfigMap" "default" "config-map-values" }} - {{- if $configMapValuesObj }} - {{- $configMapValues := index $configMapValuesObj.Object "data" }} - {{- if $configMapValues }} - {{- $valuesContent := index $configMapValues "values.yaml" }} - {{- $valuesYaml := fromYaml $valuesContent }} - {{- $resourceGroup = $valuesYaml.azure.resourceGroup | default $resourceGroup }} +{{/* Try to get storage accounts from ConfigMap if it exists */}} +{{- if (lookup "v1" "ConfigMap" .Release.Namespace "configmap-services") }} + {{- $configMap := (lookup "v1" "ConfigMap" .Release.Namespace "configmap-services").data }} + {{- range $key, $value := $configMap }} + {{- if hasPrefix "partition_storage_name_" $key }} + {{- $storageAccounts = append $storageAccounts $value }} {{- end }} {{- end }} +{{- else if .Values.blobUpload.storageAccounts }} + {{- $storageAccounts = .Values.blobUpload.storageAccounts }} {{- end }} {{- if $storageAccounts }} - {{- range $i, $storageAccount := $storageAccounts }} ---- +{{- range $i, $storageAccount := $storageAccounts }} apiVersion: v1 kind: PersistentVolume metadata: @@ -70,7 +38,7 @@ spec: storageaccount: {{ $storageAccount }} containerName: {{ $.Values.blobUpload.container }} clientID: {{ $.Values.blobUpload.clientId | default $.Values.azure.clientId }} - resourceGroup: {{ $resourceGroup }} + resourceGroup: {{ $.Values.azure.resourceGroup }} --- apiVersion: v1 kind: PersistentVolumeClaim @@ -86,7 +54,7 @@ spec: storage: 5Gi volumeName: {{ $.Release.Name }}-blob-pv-{{ $i }} --- - {{- range $.Values.blobUpload.items }} +{{- range $.Values.blobUpload.items }} apiVersion: batch/v1 kind: Job metadata: @@ -123,7 +91,7 @@ spec: - name: blob-storage mountPath: /mnt/blob restartPolicy: Never - {{- end }} {{/* end range $.Values.blobUpload.items */}} - {{- end }} {{/* end range $storageAccounts */}} -{{- end }} {{/* end if $storageAccounts */}} -{{- end }} {{/* end if .Values.blobUpload.enabled */}} \ No newline at end of file +{{- end }}{{/* end range $.Values.blobUpload.items */}} +{{- end }}{{/* end range $storageAccounts */}} +{{- end }}{{/* end if $storageAccounts */}} +{{- end }}{{/* end if .Values.blobUpload.enabled */}} \ No newline at end of file From 50e400ca54c541cafae85c7cf8919e50f54803d0 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 11 Nov 2024 08:21:49 -0600 Subject: [PATCH 77/87] Updated --- charts/osdu-developer-base/templates/storage-container-job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index 236df97a..b3c5a112 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -6,7 +6,7 @@ {{- $configMap := (lookup "v1" "ConfigMap" .Release.Namespace "configmap-services").data }} {{- range $key, $value := $configMap }} {{- if hasPrefix "partition_storage_name_" $key }} - {{- $storageAccounts = append $storageAccounts $value }} + {{- $storageAccounts = $storageAccounts | append $value }} {{- end }} {{- end }} {{- else if .Values.blobUpload.storageAccounts }} From 980a16771c90833d19219101ca80cd5b10b017ee Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 11 Nov 2024 08:30:51 -0600 Subject: [PATCH 78/87] Updated --- charts/osdu-developer-base/templates/storage-container-job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index b3c5a112..808a645e 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -6,7 +6,7 @@ {{- $configMap := (lookup "v1" "ConfigMap" .Release.Namespace "configmap-services").data }} {{- range $key, $value := $configMap }} {{- if hasPrefix "partition_storage_name_" $key }} - {{- $storageAccounts = $storageAccounts | append $value }} + {{- $_ := set $ "storageAccounts" (append $storageAccounts $value) }} {{- end }} {{- end }} {{- else if .Values.blobUpload.storageAccounts }} From d73ec6cce7ab7dcfd9495dff52552df4f00ad3c1 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 11 Nov 2024 09:20:30 -0600 Subject: [PATCH 79/87] Updated --- .../templates/storage-container-job.yaml | 174 +++++++++--------- 1 file changed, 83 insertions(+), 91 deletions(-) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index 808a645e..bd26258a 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -1,97 +1,89 @@ {{- if (default false .Values.blobUpload.enabled) -}} -{{- $storageAccounts := list }} -{{/* Try to get storage accounts from ConfigMap if it exists */}} -{{- if (lookup "v1" "ConfigMap" .Release.Namespace "configmap-services") }} - {{- $configMap := (lookup "v1" "ConfigMap" .Release.Namespace "configmap-services").data }} - {{- range $key, $value := $configMap }} - {{- if hasPrefix "partition_storage_name_" $key }} - {{- $_ := set $ "storageAccounts" (append $storageAccounts $value) }} - {{- end }} +{{/* Create resources directly for each storage account */}} +{{- $i := 0 }} +{{- range $key, $value := (lookup "v1" "ConfigMap" .Release.Namespace "configmap-services").data }} + {{- if hasPrefix "partition_storage_name_" $key }} + --- + apiVersion: v1 + kind: PersistentVolume + metadata: + annotations: + pv.kubernetes.io/provisioned-by: blob.csi.azure.com + name: {{ $.Release.Name }}-blob-pv-{{ $i }} + spec: + storageClassName: azureblob-fuse-premium + capacity: + storage: 5Gi + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + mountOptions: + - -o allow_other + - --file-cache-timeout-in-seconds=120 + csi: + driver: blob.csi.azure.com + volumeHandle: {{ $value }}_{{ $.Values.blobUpload.container }} + volumeAttributes: + storageaccount: {{ $value }} + containerName: {{ $.Values.blobUpload.container }} + clientID: {{ $.Values.blobUpload.clientId | default $.Values.azure.clientId }} + resourceGroup: {{ $.Values.azure.resourceGroup }} + --- + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: {{ $.Release.Name }}-blob-pvc-{{ $i }} + namespace: {{ $.Release.Namespace }} + spec: + storageClassName: azureblob-fuse-premium + accessModes: + - ReadWriteMany + resources: + requests: + storage: 5Gi + volumeName: {{ $.Release.Name }}-blob-pv-{{ $i }} + --- + {{- range $.Values.blobUpload.items }} + apiVersion: batch/v1 + kind: Job + metadata: + name: {{ $.Release.Name }}-blob-upload-{{ .name }}-{{ $i }} + namespace: {{ $.Release.Namespace }} + spec: + ttlSecondsAfterFinished: 300 + template: + spec: + serviceAccountName: workload-identity-sa + volumes: + - name: blob-storage + persistentVolumeClaim: + claimName: {{ $.Release.Name }}-blob-pvc-{{ $i }} + containers: + - name: blob-upload + image: mcr.microsoft.com/cbl-mariner/base/core:2.0 + command: ["/bin/sh"] + args: + - -c + - | + # Install curl + tdnf install -y curl + + # Download the file + echo "Downloading file from {{ .url }}" + curl -kso {{ .file }} "{{ .url }}" + + # Copy to mounted blob container + cp {{ .file }} /mnt/blob/{{ $.Values.blobUpload.container }}/{{ .file }} + + echo "File uploaded to container {{ $.Values.blobUpload.container }} in storage account {{ $value }}" + volumeMounts: + - name: blob-storage + mountPath: /mnt/blob + restartPolicy: Never {{- end }} -{{- else if .Values.blobUpload.storageAccounts }} - {{- $storageAccounts = .Values.blobUpload.storageAccounts }} +{{- $i = add $i 1 }} +{{- end }} {{- end }} -{{- if $storageAccounts }} -{{- range $i, $storageAccount := $storageAccounts }} -apiVersion: v1 -kind: PersistentVolume -metadata: - annotations: - pv.kubernetes.io/provisioned-by: blob.csi.azure.com - name: {{ $.Release.Name }}-blob-pv-{{ $i }} -spec: - storageClassName: azureblob-fuse-premium - capacity: - storage: 5Gi - accessModes: - - ReadWriteMany - persistentVolumeReclaimPolicy: Retain - mountOptions: - - -o allow_other - - --file-cache-timeout-in-seconds=120 - csi: - driver: blob.csi.azure.com - volumeHandle: {{ $storageAccount }}_{{ $.Values.blobUpload.container }} - volumeAttributes: - storageaccount: {{ $storageAccount }} - containerName: {{ $.Values.blobUpload.container }} - clientID: {{ $.Values.blobUpload.clientId | default $.Values.azure.clientId }} - resourceGroup: {{ $.Values.azure.resourceGroup }} ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ $.Release.Name }}-blob-pvc-{{ $i }} - namespace: {{ $.Release.Namespace }} -spec: - storageClassName: azureblob-fuse-premium - accessModes: - - ReadWriteMany - resources: - requests: - storage: 5Gi - volumeName: {{ $.Release.Name }}-blob-pv-{{ $i }} ---- -{{- range $.Values.blobUpload.items }} -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ $.Release.Name }}-blob-upload-{{ .name }}-{{ $i }} - namespace: {{ $.Release.Namespace }} -spec: - ttlSecondsAfterFinished: 300 - template: - spec: - serviceAccountName: workload-identity-sa - volumes: - - name: blob-storage - persistentVolumeClaim: - claimName: {{ $.Release.Name }}-blob-pvc-{{ $i }} - containers: - - name: blob-upload - image: mcr.microsoft.com/cbl-mariner/base/core:2.0 - command: ["/bin/sh"] - args: - - -c - - | - # Install curl - tdnf install -y curl - - # Download the file - echo "Downloading file from {{ .url }}" - curl -kso {{ .file }} "{{ .url }}" - - # Copy to mounted blob container - cp {{ .file }} /mnt/blob/{{ $.Values.blobUpload.container }}/{{ .file }} - - echo "File uploaded to container {{ $.Values.blobUpload.container }} in storage account {{ $storageAccount }}" - volumeMounts: - - name: blob-storage - mountPath: /mnt/blob - restartPolicy: Never -{{- end }}{{/* end range $.Values.blobUpload.items */}} -{{- end }}{{/* end range $storageAccounts */}} -{{- end }}{{/* end if $storageAccounts */}} {{- end }}{{/* end if .Values.blobUpload.enabled */}} \ No newline at end of file From a4af71e3e4f9d7c603be4fac817dbbe3523a0736 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 11 Nov 2024 09:37:37 -0600 Subject: [PATCH 80/87] Added target namespace. --- charts/osdu-developer-base/templates/storage-container-job.yaml | 2 +- software/applications/osdu-core/base.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/osdu-developer-base/templates/storage-container-job.yaml index bd26258a..11a9c297 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/osdu-developer-base/templates/storage-container-job.yaml @@ -2,7 +2,7 @@ {{/* Create resources directly for each storage account */}} {{- $i := 0 }} -{{- range $key, $value := (lookup "v1" "ConfigMap" .Release.Namespace "configmap-services").data }} +{{- range $key, $value := (lookup "v1" "ConfigMap" .Values.global.targetNamespace "configmap-services").data }} {{- if hasPrefix "partition_storage_name_" $key }} --- apiVersion: v1 diff --git a/software/applications/osdu-core/base.yaml b/software/applications/osdu-core/base.yaml index a22fd44f..8078f22b 100644 --- a/software/applications/osdu-core/base.yaml +++ b/software/applications/osdu-core/base.yaml @@ -24,6 +24,8 @@ spec: name: config-map-values valuesKey: values.yaml values: + global: + targetNamespace: osdu-core enableRequestAuthentication: true azure: enabled: true From 53eed7389646269374b3c88714e5466709440018 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 11 Nov 2024 09:59:09 -0600 Subject: [PATCH 81/87] Split charts --- charts/blob-upload/Chart.yaml | 9 ++++ charts/blob-upload/templates/_helpers.tpl | 52 +++++++++++++++++++ .../templates/storage-container-job.yaml | 5 +- charts/blob-upload/values.yaml | 15 ++++++ charts/osdu-developer-base/values.yaml | 17 ------ software/applications/osdu-core/base.yaml | 37 +++++++++++-- 6 files changed, 111 insertions(+), 24 deletions(-) create mode 100644 charts/blob-upload/Chart.yaml create mode 100644 charts/blob-upload/templates/_helpers.tpl rename charts/{osdu-developer-base => blob-upload}/templates/storage-container-job.yaml (94%) create mode 100644 charts/blob-upload/values.yaml diff --git a/charts/blob-upload/Chart.yaml b/charts/blob-upload/Chart.yaml new file mode 100644 index 00000000..9df11430 --- /dev/null +++ b/charts/blob-upload/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: blob-upload +type: application +description: Uploads files to Azure Blob Storage +version: 0.0.3 +appVersion: 0.0.1 +maintainers: + - name: danielscholl + url: https://github.com/azure/osdu-developer \ No newline at end of file diff --git a/charts/blob-upload/templates/_helpers.tpl b/charts/blob-upload/templates/_helpers.tpl new file mode 100644 index 00000000..7bc1d801 --- /dev/null +++ b/charts/blob-upload/templates/_helpers.tpl @@ -0,0 +1,52 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "blob-upload.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "blob-upload.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "blob-upload.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "blob-upload.labels" -}} +helm.sh/chart: {{ include "blob-upload.chart" . }} +{{ include "blob-upload.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "blob-upload.selectorLabels" -}} +app.kubernetes.io/name: {{ include "blob-upload.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} \ No newline at end of file diff --git a/charts/osdu-developer-base/templates/storage-container-job.yaml b/charts/blob-upload/templates/storage-container-job.yaml similarity index 94% rename from charts/osdu-developer-base/templates/storage-container-job.yaml rename to charts/blob-upload/templates/storage-container-job.yaml index 11a9c297..e7eabba9 100644 --- a/charts/osdu-developer-base/templates/storage-container-job.yaml +++ b/charts/blob-upload/templates/storage-container-job.yaml @@ -1,8 +1,7 @@ {{- if (default false .Values.blobUpload.enabled) -}} -{{/* Create resources directly for each storage account */}} {{- $i := 0 }} -{{- range $key, $value := (lookup "v1" "ConfigMap" .Values.global.targetNamespace "configmap-services").data }} +{{- range $key, $value := (lookup "v1" "ConfigMap" .Values.global.configNamespace "configmap-services").data }} {{- if hasPrefix "partition_storage_name_" $key }} --- apiVersion: v1 @@ -86,4 +85,4 @@ {{- end }} {{- end }} -{{- end }}{{/* end if .Values.blobUpload.enabled */}} \ No newline at end of file +{{- end }} \ No newline at end of file diff --git a/charts/blob-upload/values.yaml b/charts/blob-upload/values.yaml new file mode 100644 index 00000000..4bfa3ad2 --- /dev/null +++ b/charts/blob-upload/values.yaml @@ -0,0 +1,15 @@ + +fullnameOverride: blob-upload + +################################################################################ +# Specify the azure environment specific values +# +azure: + enabled: true + tenantId: + clientId: + keyvaultName: + +blobUpload: + enabled: false + container: "legal-service-azure-configuration" diff --git a/charts/osdu-developer-base/values.yaml b/charts/osdu-developer-base/values.yaml index 8c9b14c3..82db7a6b 100644 --- a/charts/osdu-developer-base/values.yaml +++ b/charts/osdu-developer-base/values.yaml @@ -1,20 +1,3 @@ -# Copyright © Microsoft Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for osdu-azure. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. fullnameOverride: osdu-svc diff --git a/software/applications/osdu-core/base.yaml b/software/applications/osdu-core/base.yaml index 8078f22b..16673f2b 100644 --- a/software/applications/osdu-core/base.yaml +++ b/software/applications/osdu-core/base.yaml @@ -24,8 +24,6 @@ spec: name: config-map-values valuesKey: values.yaml values: - global: - targetNamespace: osdu-core enableRequestAuthentication: true azure: enabled: true @@ -34,10 +32,41 @@ spec: defaultMemoryRequests: "1Gi" defaultCpuLimits: "2" defaultMemoryLimits: "4Gi" +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: blob-upload + namespace: default + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-developer-base-core + namespace: default + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/blob-upload + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: config-map-values + valuesKey: values.yaml + values: + global: + configmapNamespace: osdu-core + azure: + enabled: true blobUpload: enabled: true - storageAccounts: - - "partition_storage_name_0" items: - name: legal file: "Legal_COO.json" From 7ebf437ea23d29ec94e56d942160b42b192ab2d8 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 11 Nov 2024 10:06:05 -0600 Subject: [PATCH 82/87] Fixed helm chart --- charts/blob-upload/Chart.yaml | 2 +- charts/blob-upload/templates/storage-container-job.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/blob-upload/Chart.yaml b/charts/blob-upload/Chart.yaml index 9df11430..d9f9536c 100644 --- a/charts/blob-upload/Chart.yaml +++ b/charts/blob-upload/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: blob-upload type: application description: Uploads files to Azure Blob Storage -version: 0.0.3 +version: 0.0.4 appVersion: 0.0.1 maintainers: - name: danielscholl diff --git a/charts/blob-upload/templates/storage-container-job.yaml b/charts/blob-upload/templates/storage-container-job.yaml index e7eabba9..86f5c587 100644 --- a/charts/blob-upload/templates/storage-container-job.yaml +++ b/charts/blob-upload/templates/storage-container-job.yaml @@ -1,7 +1,7 @@ {{- if (default false .Values.blobUpload.enabled) -}} {{- $i := 0 }} -{{- range $key, $value := (lookup "v1" "ConfigMap" .Values.global.configNamespace "configmap-services").data }} +{{- range $key, $value := (lookup "v1" "ConfigMap" .Values.global.configmapNamespace "configmap-services").data }} {{- if hasPrefix "partition_storage_name_" $key }} --- apiVersion: v1 From f3eb5fd31511b519c11e64bfbbb1e78418fa1a43 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 11 Nov 2024 16:26:18 -0600 Subject: [PATCH 83/87] Updated --- bicep/modules/blade_partition.bicep | 2 + .../templates/storage-container-job.yaml | 142 +++++++++--------- charts/blob-upload/values.yaml | 9 +- software/applications/osdu-core/base.yaml | 2 - 4 files changed, 78 insertions(+), 77 deletions(-) diff --git a/bicep/modules/blade_partition.bicep b/bicep/modules/blade_partition.bicep index b02b8944..efe84afe 100644 --- a/bicep/modules/blade_partition.bicep +++ b/bicep/modules/blade_partition.bicep @@ -513,6 +513,8 @@ module storage 'storage-account/main.bicep' = [for (partition, index) in partiti } ] + enableHierarchicalNamespace: true + // Apply Security allowBlobPublicAccess: enableBlobPublicAccess publicNetworkAccess: 'Enabled' diff --git a/charts/blob-upload/templates/storage-container-job.yaml b/charts/blob-upload/templates/storage-container-job.yaml index 86f5c587..fd01829e 100644 --- a/charts/blob-upload/templates/storage-container-job.yaml +++ b/charts/blob-upload/templates/storage-container-job.yaml @@ -1,88 +1,90 @@ -{{- if (default false .Values.blobUpload.enabled) -}} - +{{- if (default false .Values.blobUpload.enabled) }} {{- $i := 0 }} {{- range $key, $value := (lookup "v1" "ConfigMap" .Values.global.configmapNamespace "configmap-services").data }} - {{- if hasPrefix "partition_storage_name_" $key }} - --- - apiVersion: v1 - kind: PersistentVolume - metadata: - annotations: - pv.kubernetes.io/provisioned-by: blob.csi.azure.com - name: {{ $.Release.Name }}-blob-pv-{{ $i }} +{{- if hasPrefix "partition_storage_name_" $key }} +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + annotations: + pv.kubernetes.io/provisioned-by: blob.csi.azure.com + name: {{ $.Release.Name }}-blob-pv-{{ $i }} +spec: + storageClassName: azureblob-nfs-premium + capacity: + storage: 1Gi + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + mountOptions: + - vers=3 + - nolock + - sec=sys + - allow_other + - timeo=120 + csi: + driver: blob.csi.azure.com + readOnly: false + volumeHandle: {{ $value }}##{{ $.Values.blobUpload.container }} + volumeAttributes: + resourceGroup: {{ $.Values.azure.resourceGroup }} + storageAccount: {{ $value }} + containerName: {{ $.Values.blobUpload.container }} + protocol: nfs + AzureStorageAuthType: msi + AzureStorageIdentityClientID: {{ $.Values.azure.clientId }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ $.Release.Name }}-blob-pvc-{{ $i }} + namespace: {{ $.Release.Namespace }} +spec: + storageClassName: azureblob-nfs-premium + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi + volumeName: {{ $.Release.Name }}-blob-pv-{{ $i }} +--- +{{- range $.Values.blobUpload.items }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $.Release.Name }}-blob-upload-{{ .name }}-{{ $i }} + namespace: {{ $.Release.Namespace }} +spec: + ttlSecondsAfterFinished: 300 + template: spec: - storageClassName: azureblob-fuse-premium - capacity: - storage: 5Gi - accessModes: - - ReadWriteMany - persistentVolumeReclaimPolicy: Retain - mountOptions: - - -o allow_other - - --file-cache-timeout-in-seconds=120 - csi: - driver: blob.csi.azure.com - volumeHandle: {{ $value }}_{{ $.Values.blobUpload.container }} - volumeAttributes: - storageaccount: {{ $value }} - containerName: {{ $.Values.blobUpload.container }} - clientID: {{ $.Values.blobUpload.clientId | default $.Values.azure.clientId }} - resourceGroup: {{ $.Values.azure.resourceGroup }} - --- - apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - name: {{ $.Release.Name }}-blob-pvc-{{ $i }} - namespace: {{ $.Release.Namespace }} - spec: - storageClassName: azureblob-fuse-premium - accessModes: - - ReadWriteMany - resources: - requests: - storage: 5Gi - volumeName: {{ $.Release.Name }}-blob-pv-{{ $i }} - --- - {{- range $.Values.blobUpload.items }} - apiVersion: batch/v1 - kind: Job - metadata: - name: {{ $.Release.Name }}-blob-upload-{{ .name }}-{{ $i }} - namespace: {{ $.Release.Namespace }} - spec: - ttlSecondsAfterFinished: 300 - template: - spec: - serviceAccountName: workload-identity-sa - volumes: - - name: blob-storage - persistentVolumeClaim: - claimName: {{ $.Release.Name }}-blob-pvc-{{ $i }} - containers: - - name: blob-upload - image: mcr.microsoft.com/cbl-mariner/base/core:2.0 - command: ["/bin/sh"] - args: + # Uncomment if using workload identity + serviceAccountName: workload-identity-sa + volumes: + - name: blob-storage + persistentVolumeClaim: + claimName: {{ $.Release.Name }}-blob-pvc-{{ $i }} + containers: + - name: blob-upload + image: mcr.microsoft.com/cbl-mariner/base/core:2.0 + command: ["/bin/sh"] + args: - -c - | # Install curl tdnf install -y curl - # Download the file echo "Downloading file from {{ .url }}" curl -kso {{ .file }} "{{ .url }}" - # Copy to mounted blob container - cp {{ .file }} /mnt/blob/{{ $.Values.blobUpload.container }}/{{ .file }} - + cp {{ .file }} /mnt/blob/{{ .file }} echo "File uploaded to container {{ $.Values.blobUpload.container }} in storage account {{ $value }}" - volumeMounts: + volumeMounts: - name: blob-storage mountPath: /mnt/blob - restartPolicy: Never - {{- end }} + restartPolicy: Never +{{- end }} {{- $i = add $i 1 }} {{- end }} {{- end }} - {{- end }} \ No newline at end of file diff --git a/charts/blob-upload/values.yaml b/charts/blob-upload/values.yaml index 4bfa3ad2..66b0e46a 100644 --- a/charts/blob-upload/values.yaml +++ b/charts/blob-upload/values.yaml @@ -1,14 +1,13 @@ fullnameOverride: blob-upload -################################################################################ -# Specify the azure environment specific values -# +global: + configmapNamespace: osdu-core + azure: - enabled: true tenantId: clientId: - keyvaultName: + resourceGroup: blobUpload: enabled: false diff --git a/software/applications/osdu-core/base.yaml b/software/applications/osdu-core/base.yaml index 16673f2b..8466b94a 100644 --- a/software/applications/osdu-core/base.yaml +++ b/software/applications/osdu-core/base.yaml @@ -63,8 +63,6 @@ spec: values: global: configmapNamespace: osdu-core - azure: - enabled: true blobUpload: enabled: true items: From 0485996bf748a3ad9c77c5f2c5a0962523e2e884 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 11 Nov 2024 17:34:03 -0600 Subject: [PATCH 84/87] Updated Chart --- charts/blob-upload/Chart.yaml | 2 +- .../templates/storage-container-job.yaml | 80 +++++-------------- charts/blob-upload/values.yaml | 8 +- 3 files changed, 23 insertions(+), 67 deletions(-) diff --git a/charts/blob-upload/Chart.yaml b/charts/blob-upload/Chart.yaml index d9f9536c..28fbe399 100644 --- a/charts/blob-upload/Chart.yaml +++ b/charts/blob-upload/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: blob-upload type: application description: Uploads files to Azure Blob Storage -version: 0.0.4 +version: 0.0.1 appVersion: 0.0.1 maintainers: - name: danielscholl diff --git a/charts/blob-upload/templates/storage-container-job.yaml b/charts/blob-upload/templates/storage-container-job.yaml index fd01829e..a28fd820 100644 --- a/charts/blob-upload/templates/storage-container-job.yaml +++ b/charts/blob-upload/templates/storage-container-job.yaml @@ -2,53 +2,8 @@ {{- $i := 0 }} {{- range $key, $value := (lookup "v1" "ConfigMap" .Values.global.configmapNamespace "configmap-services").data }} {{- if hasPrefix "partition_storage_name_" $key }} ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - annotations: - pv.kubernetes.io/provisioned-by: blob.csi.azure.com - name: {{ $.Release.Name }}-blob-pv-{{ $i }} -spec: - storageClassName: azureblob-nfs-premium - capacity: - storage: 1Gi - accessModes: - - ReadWriteMany - persistentVolumeReclaimPolicy: Retain - mountOptions: - - vers=3 - - nolock - - sec=sys - - allow_other - - timeo=120 - csi: - driver: blob.csi.azure.com - readOnly: false - volumeHandle: {{ $value }}##{{ $.Values.blobUpload.container }} - volumeAttributes: - resourceGroup: {{ $.Values.azure.resourceGroup }} - storageAccount: {{ $value }} - containerName: {{ $.Values.blobUpload.container }} - protocol: nfs - AzureStorageAuthType: msi - AzureStorageIdentityClientID: {{ $.Values.azure.clientId }} ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ $.Release.Name }}-blob-pvc-{{ $i }} - namespace: {{ $.Release.Namespace }} -spec: - storageClassName: azureblob-nfs-premium - accessModes: - - ReadWriteMany - resources: - requests: - storage: 1Gi - volumeName: {{ $.Release.Name }}-blob-pv-{{ $i }} ---- {{- range $.Values.blobUpload.items }} +--- apiVersion: batch/v1 kind: Job metadata: @@ -57,34 +12,41 @@ metadata: spec: ttlSecondsAfterFinished: 300 template: + metadata: + labels: + azure.workload.identity/use: "true" + annotations: + azure.workload.identity/client-id: {{ $.Values.azure.clientId | quote }} + azure.workload.identity/tenant-id: {{ $.Values.azure.tenantId | quote }} spec: - # Uncomment if using workload identity serviceAccountName: workload-identity-sa - volumes: - - name: blob-storage - persistentVolumeClaim: - claimName: {{ $.Release.Name }}-blob-pvc-{{ $i }} containers: - name: blob-upload - image: mcr.microsoft.com/cbl-mariner/base/core:2.0 - command: ["/bin/sh"] + image: mcr.microsoft.com/azure-cli:cbl-mariner2.0-amd64 + command: ["/bin/bash"] args: - -c - | # Install curl tdnf install -y curl + # Download the file echo "Downloading file from {{ .url }}" curl -kso {{ .file }} "{{ .url }}" - # Copy to mounted blob container - cp {{ .file }} /mnt/blob/{{ .file }} + + # Upload directly to blob storage using Azure CLI + az login --identity + az storage blob upload \ + -f {{ .file }} \ + -c {{ $.Values.blobUpload.container }} \ + -n {{ .file }} \ + --overwrite \ + --auth-mode login + echo "File uploaded to container {{ $.Values.blobUpload.container }} in storage account {{ $value }}" - volumeMounts: - - name: blob-storage - mountPath: /mnt/blob restartPolicy: Never {{- end }} {{- $i = add $i 1 }} {{- end }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} \ No newline at end of file diff --git a/charts/blob-upload/values.yaml b/charts/blob-upload/values.yaml index 66b0e46a..b0764fb5 100644 --- a/charts/blob-upload/values.yaml +++ b/charts/blob-upload/values.yaml @@ -1,14 +1,8 @@ - fullnameOverride: blob-upload global: configmapNamespace: osdu-core -azure: - tenantId: - clientId: - resourceGroup: - blobUpload: - enabled: false + enabled: true container: "legal-service-azure-configuration" From bbf14ee2fa396a64bfeb2ebc5118ee0c71c64fc7 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 11 Nov 2024 17:44:30 -0600 Subject: [PATCH 85/87] Updated Chart --- charts/blob-upload/templates/storage-container-job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/blob-upload/templates/storage-container-job.yaml b/charts/blob-upload/templates/storage-container-job.yaml index a28fd820..50edca35 100644 --- a/charts/blob-upload/templates/storage-container-job.yaml +++ b/charts/blob-upload/templates/storage-container-job.yaml @@ -35,7 +35,7 @@ spec: curl -kso {{ .file }} "{{ .url }}" # Upload directly to blob storage using Azure CLI - az login --identity + az login --identity --username {{ $.Values.azure.clientId }} az storage blob upload \ -f {{ .file }} \ -c {{ $.Values.blobUpload.container }} \ From e89255285f000268615750384fbb4bfda286a7f7 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 11 Nov 2024 18:00:42 -0600 Subject: [PATCH 86/87] Updated Chart --- charts/blob-upload/Chart.yaml | 2 +- charts/blob-upload/templates/storage-container-job.yaml | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/charts/blob-upload/Chart.yaml b/charts/blob-upload/Chart.yaml index 28fbe399..76c69e0c 100644 --- a/charts/blob-upload/Chart.yaml +++ b/charts/blob-upload/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: blob-upload type: application description: Uploads files to Azure Blob Storage -version: 0.0.1 +version: 0.0.2 appVersion: 0.0.1 maintainers: - name: danielscholl diff --git a/charts/blob-upload/templates/storage-container-job.yaml b/charts/blob-upload/templates/storage-container-job.yaml index 50edca35..71154081 100644 --- a/charts/blob-upload/templates/storage-container-job.yaml +++ b/charts/blob-upload/templates/storage-container-job.yaml @@ -34,8 +34,13 @@ spec: echo "Downloading file from {{ .url }}" curl -kso {{ .file }} "{{ .url }}" + # Login using workload identity + az login --service-principal \ + --username {{ $.Values.azure.clientId }} \ + --tenant {{ $.Values.azure.tenantId }} \ + --identity + # Upload directly to blob storage using Azure CLI - az login --identity --username {{ $.Values.azure.clientId }} az storage blob upload \ -f {{ .file }} \ -c {{ $.Values.blobUpload.container }} \ From e7a709c9bef9f6add961ec71aaafc853bf3ab529 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 11 Nov 2024 18:42:43 -0600 Subject: [PATCH 87/87] Updated --- bicep/main.bicep | 25 +++---- bicep/modules/blade_partition.bicep | 52 +++++++------- .../templates/storage-container-job.yaml | 12 +--- software/applications/osdu-core/base.yaml | 72 +++++++++---------- 4 files changed, 77 insertions(+), 84 deletions(-) diff --git a/bicep/main.bicep b/bicep/main.bicep index 27f1f903..dc0b73a0 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -948,18 +948,19 @@ module gitOpsUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for } }] -module storageAcl 'modules/network_acl_storage.bicep' = { - name: '${configuration.name}-storage-acl' - params: { - storageName: storage.outputs.name - location: location - skuName: configuration.storage.sku - natClusterIP: clusterBlade.outputs.natClusterIP - } - dependsOn: [ - gitOpsUpload - ] -} +//TODO: This can't be done yet. +// module storageAcl 'modules/network_acl_storage.bicep' = { +// name: '${configuration.name}-storage-acl' +// params: { +// storageName: storage.outputs.name +// location: location +// skuName: configuration.storage.sku +// natClusterIP: clusterBlade.outputs.natClusterIP +// } +// dependsOn: [ +// gitOpsUpload +// ] +// } /* diff --git a/bicep/modules/blade_partition.bicep b/bicep/modules/blade_partition.bicep index efe84afe..8df7b7e9 100644 --- a/bicep/modules/blade_partition.bicep +++ b/bicep/modules/blade_partition.bicep @@ -671,34 +671,34 @@ module partitonNamespace 'br/public:avm/res/service-bus/namespace:0.9.1' = [for // TODO: This should be moved to the Kubernetes Job. -// module blobUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for (partition, index) in partitions: { -// name: '${bladeConfig.sectionName}-storage-blob-upload-${index}' -// params: { -// name: 'script-${storage[index].outputs.name}-Legal_COO' -// location: location -// cleanupPreference: 'Always' -// retentionInterval: 'PT1H' -// timeout: 'PT30M' -// runOnce: true +module blobUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for (partition, index) in partitions: { + name: '${bladeConfig.sectionName}-storage-blob-upload-${index}' + params: { + name: 'script-${storage[index].outputs.name}-Legal_COO' + location: location + cleanupPreference: 'Always' + retentionInterval: 'PT1H' + timeout: 'PT30M' + runOnce: true -// managedIdentities: { -// userAssignedResourcesIds: [ -// stampIdentity.id -// ] -// } - -// kind: 'AzureCLI' -// azCliVersion: '2.63.0' + managedIdentities: { + userAssignedResourcesIds: [ + stampIdentity.id + ] + } + + kind: 'AzureCLI' + azCliVersion: '2.63.0' -// environmentVariables: [ -// { name: 'CONTENT', value: loadTextContent('./deploy-scripts/Legal_COO.json') } -// { name: 'FILE_NAME', value: 'Legal_COO.json' } -// { name: 'CONTAINER', value: 'legal-service-azure-configuration' } -// { name: 'AZURE_STORAGE_ACCOUNT', value: storage[index].outputs.name } -// ] -// scriptContent: loadTextContent('./deploy-scripts/blob_upload.sh') -// } -// }] + environmentVariables: [ + { name: 'CONTENT', value: loadTextContent('./deploy-scripts/Legal_COO.json') } + { name: 'FILE_NAME', value: 'Legal_COO.json' } + { name: 'CONTAINER', value: 'legal-service-azure-configuration' } + { name: 'AZURE_STORAGE_ACCOUNT', value: storage[index].outputs.name } + ] + scriptContent: loadTextContent('./deploy-scripts/blob_upload.sh') + } +}] // TODO: ACL can only be applied after the blob upload. diff --git a/charts/blob-upload/templates/storage-container-job.yaml b/charts/blob-upload/templates/storage-container-job.yaml index 71154081..c2f0e658 100644 --- a/charts/blob-upload/templates/storage-container-job.yaml +++ b/charts/blob-upload/templates/storage-container-job.yaml @@ -12,12 +12,6 @@ metadata: spec: ttlSecondsAfterFinished: 300 template: - metadata: - labels: - azure.workload.identity/use: "true" - annotations: - azure.workload.identity/client-id: {{ $.Values.azure.clientId | quote }} - azure.workload.identity/tenant-id: {{ $.Values.azure.tenantId | quote }} spec: serviceAccountName: workload-identity-sa containers: @@ -35,10 +29,7 @@ spec: curl -kso {{ .file }} "{{ .url }}" # Login using workload identity - az login --service-principal \ - --username {{ $.Values.azure.clientId }} \ - --tenant {{ $.Values.azure.tenantId }} \ - --identity + az login --identity # Upload directly to blob storage using Azure CLI az storage blob upload \ @@ -49,6 +40,7 @@ spec: --auth-mode login echo "File uploaded to container {{ $.Values.blobUpload.container }} in storage account {{ $value }}" + sleep 300000 restartPolicy: Never {{- end }} {{- $i = add $i 1 }} diff --git a/software/applications/osdu-core/base.yaml b/software/applications/osdu-core/base.yaml index 8466b94a..0741db96 100644 --- a/software/applications/osdu-core/base.yaml +++ b/software/applications/osdu-core/base.yaml @@ -33,39 +33,39 @@ spec: defaultCpuLimits: "2" defaultMemoryLimits: "4Gi" --- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: blob-upload - namespace: default - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-developer-base-core - namespace: default - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/blob-upload - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - install: - remediation: - retries: 3 - valuesFrom: - - kind: ConfigMap - name: config-map-values - valuesKey: values.yaml - values: - global: - configmapNamespace: osdu-core - blobUpload: - enabled: true - items: - - name: legal - file: "Legal_COO.json" - url: "https://raw.githubusercontent.com/Azure/osdu-developer/refs/heads/main/bicep/modules/script-blob-upload/Legal_COO.json" +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: blob-upload +# namespace: default +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-developer-base-core +# namespace: default +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/blob-upload +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# install: +# remediation: +# retries: 3 +# valuesFrom: +# - kind: ConfigMap +# name: config-map-values +# valuesKey: values.yaml +# values: +# global: +# configmapNamespace: osdu-core +# blobUpload: +# enabled: true +# items: +# - name: legal +# file: "Legal_COO.json" +# url: "https://raw.githubusercontent.com/Azure/osdu-developer/refs/heads/main/bicep/modules/script-blob-upload/Legal_COO.json"