如何解决HDInsight Azure adls gen2'InternalServerError'ARM模板部署
使用ADLS Gen 2创建具有用户分配的具有StorageBlobdataOwner角色的托管idnetity的Azure HDinsight Spark群集。 已成功将msi角色分配给存储，但HDInsight部署出错（内部服务器错误） 我认为模板中的HDInsight群集（存储配置文件）资源代码附近存在一些问题。我可以在这里使用一些帮助。下面是图片。
{ “ $ schema”：“ https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#”，
"contentVersion": "1.0.0.0","parameters": {
"clusterType": {
"type": "string","allowedValues": [ "hadoop","hbase","storm","spark" ],"Metadata": {
"description": "The type of the HDInsight cluster to create."
}
},"clusterName": {
"type": "string","Metadata": {
"description": "The name of the HDInsight cluster to create."
}
},"clusterLoginUserName": {
"type": "string","Metadata": {
"description": "These credentials can be used to submit jobs to the cluster and to log into cluster dashboards."
}
},"clusterLoginPassword": {
"type": "securestring","minLength": 10,"Metadata": {
"description": "The clusterloginpassword must be at least 10 characters in length and must contain at least one digit,one upper case letter,one lower case letter,and one non-alphanumeric character except (single-quote,double-quote,backslash,right-bracket,full-stop). Also,the password must not contain 3 consecutive characters from the cluster username or SSH username."
}
},"sshUserName": {
"type": "string","Metadata": {
"description": "These credentials can be used to remotely access the cluster and should not be same as clusterLoginUserName."
}
},"sshPassword": {
"type": "securestring","minLength": 6,"maxLength": 72,"Metadata": {
"description": "SSH password must be 6-72 characters long and must contain at least one digit,and one lower case letter. It must not contain any 3 consecutive characters from the cluster login name"
}
},"location": {
"type": "string","defaultValue": "[resourceGroup().location]","Metadata": {
"description": "Location for all resources."
}
},"HeadNodeVirtualMachinesize": {
"type": "string","defaultValue": "Standard_D12_v2","allowedValues": [
"Standard_A4_v2","Standard_A8_v2","Standard_D3_v2","Standard_D4_v2","Standard_D5_v2","Standard_D12_v2","Standard_D13_v2"
],"Metadata": {
"description": "This is the headnode Azure Virtual Machine size,and will affect the cost. If you don't kNow,just leave the default value."
}
},"WorkerNodeVirtualMachinesize": {
"type": "string","defaultValue": "Standard_D13_v2","Standard_D1_v2","Standard_D2_v2","Metadata": {
"description": "This is the workerdnode Azure Virtual Machine size,"clusterHeadNodeCount": {
"type": "int","defaultValue": 2,"Metadata": {
"description": "Number of worker nodes"
}
},"clusterWorkerNodeCount": {
"type": "int","defaultValue": 4,"StorageAccountName": {
"type": "string","Metadata": {
"description": "Name of the Storage Account"
}
},"StorageAccountType": {
"type": "string","defaultValue": "Standard_lrs","allowedValues": [
"Standard_lrs","Standard_GRS","Standard_ZRS","Standard_RA-GRS"
],"Metadata": {
"description": "Type of the Storage Account"
}
},"filesystemname": {
"type": "string","Metadata": {
"description": "Name of the container"
}
},"UserAssignedIdentityName": {
"type": "string","Metadata": {
"description": "Name of the User Assigned Identity"
}
}
},"variables": {
"managedIdentityId": "[concat('/subscriptions/',subscription().subscriptionId,'/resourceGroups/',resourceGroup().name,'/providers/Microsoft.ManagedIdentity/userAssignedIdentities/',parameters('UserAssignedIdentityName'))]","StorageApiVersion": "2019-06-01","msiApiVersion": "2018-11-30","HDInsightApiVersion": "2015-03-01-preview","StorageBlobDataOwner": "[concat('/subscriptions/','/providers/Microsoft.Authorization/roleDeFinitions/','b7e6dc6d-f1e8-4753-8033-0f276bb0955b')]","StorageBlobDataContributor": "[concat('/subscriptions/','ba92f5b4-2d11-453d-a403-e96b0029c9fe')]"
},"resources": [
{
"name": "[parameters('UserAssignedIdentityName')]","type": "Microsoft.ManagedIdentity/userAssignedIdentities","apiVersion": "[variables('msiApiVersion')]","location": "[resourceGroup().location]"
},{
"type": "Microsoft.Storage/storageAccounts","apiVersion": "[variables('StorageApiVersion')]","name": "[parameters('StorageAccountName')]","location": "[parameters('location')]","sku": {
"name": "[parameters('StorageAccountType')]"
},"kind": "StorageV2","properties": {
"encryption": {
"keySource": "Microsoft.Storage","services": {
"blob": {
"enabled": true
},"file": {
"enabled": true
}
}
},"isHnsEnabled": true,"supportsHttpsTrafficOnly": true
}
},{
"type": "Microsoft.Storage/storageAccounts/providers/roleAssignments","apiVersion": "2018-01-01-preview","name": "[concat(parameters('StorageAccountName'),'/Microsoft.Authorization/',guid(subscription().subscriptionId))]","dependsOn": [
"[resourceId('Microsoft.Storage/storageAccounts',parameters('StorageAccountName'))]","[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities',parameters('UserAssignedIdentityName'))]"
],"properties": {
"roleDeFinitionId": "[variables('StorageBlobDataOwner')]","principalId": "[reference(variables('managedIdentityId'),variables('msiApiVersion')).principalId]"
}
},{
"apiVersion": "[variables('HDInsightApiVersion')]","name": "[parameters('clusterName')]","type": "Microsoft.HDInsight/clusters","properties": {
"clusterVersion": "4.0","osType": "Linux","tier": "standard","clusterDeFinition": {
"kind": "[parameters('clusterType')]","componentVersion": {
"Spark": "2.3"
},"configurations": {
"gateway": {
"restAuthCredential.isEnabled": true,"restAuthCredential.username": "[parameters('clusterLoginUserName')]","restAuthCredential.password": "[parameters('clusterLoginPassword')]"
}
}
},"identity": {
"type": "UserAssigned","userAssignedIdentities": {
"[variables('managedIdentityId')]": {}
}
},"storageProfile": {
"storageaccounts": [
{
"name": "[concat(parameters('StorageAccountName'),'.blob.core.windows.net')]","isDefault": true,"fileSystem": "[parameters('filesystemname')]","resourceId": "[reference(resourceId('Microsoft.Storage/storageAccounts',parameters('StorageAccountName')),variables('StorageApiVersion'))]","msiResourceId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities',parameters('UserAssignedIdentityName')),variables('msiApiVersion'))]"
}
]
},"computeProfile": {
"roles": [
{
"name": "headnode","minInstanceCount": 1,"targetInstanceCount": "[parameters('clusterHeadNodeCount')]","hardwareProfile": {
"vmSize": "[parameters('HeadNodeVirtualMachinesize')]"
},"osProfile": {
"linuxOperatingSystemProfile": {
"username": "[parameters('sshUserName')]","password": "[parameters('sshPassword')]"
}
},"virtualNetworkProfile": null,"scriptActions": []
},{
"name": "workernode","targetInstanceCount": "[parameters('clusterWorkerNodeCount')]","autoscale": {
"capacity": {
"minInstanceCount": 3,"maxInstanceCount": 10
}
},"hardwareProfile": {
"vmSize": "[parameters('WorkerNodeVirtualMachinesize')]"
},"scriptActions": []
}
]
}
}
}
],"outputs": {
"storage": {
"type": "object","value": "[reference(resourceId('Microsoft.Storage/storageAccounts',parameters('StorageAccountName')))]"
},"cluster": {
"type": "object","value": "[reference(resourceId('Microsoft.HDInsight/clusters',parameters('clusterName')))]"
}
}
}
InternalServerError and Operation detail shows "Anerror has occured" and no other info
解决方法
更新：确保您的存储帐户具有用户分配的身份，并具有Storage Blob Data Contributor个角色权限，否则群集创建将失败。
如果您正在使用Azure Data Lake Storage Gen2并收到错误AmbariClusterCreationFailedErrorCode: "Internal server error occurred while processing the request. Please retry the request or contact support."。
要解决此问题，请打开Azure门户，转到您的存储帐户，然后在访问控制（IAM）下，确保Storage Blob Data Contributor或Storage Blob Data Owner role拥有对“用户分配的受管用户”的“分配的访问权限”订阅的身份。有关详细说明，请参见Set up permissions for the managed identity on the Data Lake Storage Gen2 account。
确保已按照必要的步骤配置了Data Lake Storage gen2帐户。
参考： Use Azure Data Lake Storage Gen2 with Azure HDInsight clusters
版权声明：本文内容由互联网用户自发贡献，该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 dio@foxmail.com 举报，一经查实，本站将立刻删除。
