Creating Azure HDinsight Spark cluster with ADLS Gen 2,Userassigned managed idnetity with StorageBlobdataOwner role. Successfully assigned msi role to storage but getting error with HDInsight deployment(Internal server error) Theres some issue near HDInsight cluster(Storage profile)resource code in the template i think. I could use some help here.Attached image below.
{ "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"clusterType": {
"type": "string",
"allowedValues": [ "hadoop", "hbase", "storm", "spark" ],
"metadata": {
"description": "The type of the HDInsight cluster to create."
}
},
"clusterName": {
"type": "string",
"metadata": {
"description": "The name of the HDInsight cluster to create."
}
},
"clusterLoginUserName": {
"type": "string",
"metadata": {
"description": "These credentials can be used to submit jobs to the cluster and to log into cluster dashboards."
}
},
"clusterLoginPassword": {
"type": "securestring",
"minLength": 10,
"metadata": {
"description": "The clusterloginpassword must be at least 10 characters in length and must contain at least one digit, one upper case letter, one lower case letter, and one non-alphanumeric character except (single-quote, double-quote, backslash, right-bracket, full-stop). Also, the password must not contain 3 consecutive characters from the cluster username or SSH username."
}
},
"sshUserName": {
"type": "string",
"metadata": {
"description": "These credentials can be used to remotely access the cluster and should not be same as clusterLoginUserName."
}
},
"sshPassword": {
"type": "securestring",
"minLength": 6,
"maxLength": 72,
"metadata": {
"description": "SSH password must be 6-72 characters long and must contain at least one digit, one upper case letter, and one lower case letter. It must not contain any 3 consecutive characters from the cluster login name"
}
},
"location": {
"type": "string",
"defaultValue": "[resourceGroup().location]",
"metadata": {
"description": "Location for all resources."
}
},
"HeadNodeVirtualMachineSize": {
"type": "string",
"defaultValue": "Standard_D12_v2",
"allowedValues": [
"Standard_A4_v2",
"Standard_A8_v2",
"Standard_D3_v2",
"Standard_D4_v2",
"Standard_D5_v2",
"Standard_D12_v2",
"Standard_D13_v2"
],
"metadata": {
"description": "This is the headnode Azure Virtual Machine size, and will affect the cost. If you don't know, just leave the default value."
}
},
"WorkerNodeVirtualMachineSize": {
"type": "string",
"defaultValue": "Standard_D13_v2",
"allowedValues": [
"Standard_A4_v2",
"Standard_A8_v2",
"Standard_D1_v2",
"Standard_D2_v2",
"Standard_D3_v2",
"Standard_D4_v2",
"Standard_D5_v2",
"Standard_D12_v2",
"Standard_D13_v2"
],
"metadata": {
"description": "This is the workerdnode Azure Virtual Machine size, and will affect the cost. If you don't know, just leave the default value."
}
},
"clusterHeadNodeCount": {
"type": "int",
"defaultValue": 2,
"metadata": {
"description": "Number of worker nodes"
}
},
"clusterWorkerNodeCount": {
"type": "int",
"defaultValue": 4,
"metadata": {
"description": "Number of worker nodes"
}
},
"StorageAccountName": {
"type": "string",
"metadata": {
"description": "Name of the Storage Account"
}
},
"StorageAccountType": {
"type": "string",
"defaultValue": "Standard_LRS",
"allowedValues": [
"Standard_LRS",
"Standard_GRS",
"Standard_ZRS",
"Standard_RA-GRS"
],
"metadata": {
"description": "Type of the Storage Account"
}
},
"filesystemname": {
"type": "string",
"metadata": {
"description": "Name of the container"
}
},
"UserAssignedIdentityName": {
"type": "string",
"metadata": {
"description": "Name of the User Assigned Identity"
}
}
},
"variables": {
"managedIdentityId": "[concat('/subscriptions/', subscription().subscriptionId, '/resourceGroups/',resourceGroup().name, '/providers/Microsoft.ManagedIdentity/userAssignedIdentities/', parameters('UserAssignedIdentityName'))]",
"StorageApiVersion": "2019-06-01",
"msiApiVersion": "2018-11-30",
"HDInsightApiVersion": "2015-03-01-preview",
"StorageBlobDataOwner": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'b7e6dc6d-f1e8-4753-8033-0f276bb0955b')]",
"StorageBlobDataContributor": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe')]"
},
"resources": [
{
"name": "[parameters('UserAssignedIdentityName')]",
"type": "Microsoft.ManagedIdentity/userAssignedIdentities",
"apiVersion": "[variables('msiApiVersion')]",
"location": "[resourceGroup().location]"
},
{
"type": "Microsoft.Storage/storageAccounts",
"apiVersion": "[variables('StorageApiVersion')]",
"name": "[parameters('StorageAccountName')]",
"location": "[parameters('location')]",
"sku": {
"name": "[parameters('StorageAccountType')]"
},
"kind": "StorageV2",
"properties": {
"encryption": {
"keySource": "Microsoft.Storage",
"services": {
"blob": {
"enabled": true
},
"file": {
"enabled": true
}
}
},
"isHnsEnabled": true,
"supportsHttpsTrafficOnly": true
}
},
{
"type": "Microsoft.Storage/storageAccounts/providers/roleAssignments",
"apiVersion": "2018-01-01-preview",
"name": "[concat(parameters('StorageAccountName'),'/Microsoft.Authorization/',guid(subscription().subscriptionId))]",
"dependsOn": [
"[resourceId('Microsoft.Storage/storageAccounts',parameters('StorageAccountName'))]",
"[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities',parameters('UserAssignedIdentityName'))]"
],
"properties": {
"roleDefinitionId": "[variables('StorageBlobDataOwner')]",
"principalId": "[reference(variables('managedIdentityId'),variables('msiApiVersion')).principalId]"
}
},
{
"apiVersion": "[variables('HDInsightApiVersion')]",
"name": "[parameters('clusterName')]",
"type": "Microsoft.HDInsight/clusters",
"location": "[parameters('location')]",
"dependsOn": [
"[resourceId('Microsoft.Storage/storageAccounts',parameters('StorageAccountName'))]",
"[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities',parameters('UserAssignedIdentityName'))]"
],
"properties": {
"clusterVersion": "4.0",
"osType": "Linux",
"tier": "standard",
"clusterDefinition": {
"kind": "[parameters('clusterType')]",
"componentVersion": {
"Spark": "2.3"
},
"configurations": {
"gateway": {
"restAuthCredential.isEnabled": true,
"restAuthCredential.username": "[parameters('clusterLoginUserName')]",
"restAuthCredential.password": "[parameters('clusterLoginPassword')]"
}
}
},
"identity": {
"type": "UserAssigned",
"userAssignedIdentities": {
"[variables('managedIdentityId')]": {}
}
},
"storageProfile": {
"storageaccounts": [
{
"name": "[concat(parameters('StorageAccountName'),'.blob.core.windows.net')]",
"isDefault": true,
"fileSystem": "[parameters('filesystemname')]",
"resourceId": "[reference(resourceId('Microsoft.Storage/storageAccounts',parameters('StorageAccountName')),variables('StorageApiVersion'))]",
"msiResourceId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities',parameters('UserAssignedIdentityName')),variables('msiApiVersion'))]"
}
]
},
"computeProfile": {
"roles": [
{
"name": "headnode",
"minInstanceCount": 1,
"targetInstanceCount": "[parameters('clusterHeadNodeCount')]",
"hardwareProfile": {
"vmSize": "[parameters('HeadNodeVirtualMachineSize')]"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "[parameters('sshUserName')]",
"password": "[parameters('sshPassword')]"
}
},
"virtualNetworkProfile": null,
"scriptActions": []
},
{
"name": "workernode",
"targetInstanceCount": "[parameters('clusterWorkerNodeCount')]",
"autoscale": {
"capacity": {
"minInstanceCount": 3,
"maxInstanceCount": 10
}
},
"hardwareProfile": {
"vmSize": "[parameters('WorkerNodeVirtualMachineSize')]"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "[parameters('sshUserName')]",
"password": "[parameters('sshPassword')]"
}
},
"virtualNetworkProfile": null,
"scriptActions": []
}
]
}
}
}
],
"outputs": {
"storage": {
"type": "object",
"value": "[reference(resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccountName')))]"
},
"cluster": {
"type": "object",
"value": "[reference(resourceId('Microsoft.HDInsight/clusters', parameters('clusterName')))]"
}
}
}
InternalServerError and Operation detail shows "Anerror has occured" and no other info
