azdo-databricks/arm/azure-data-factory-linkedservice-databricks-msi.json at master · pratikk-databricks/azdo-databricks · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
{
  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
  "contentVersion": "1.0.0.0",
  "parameters": {
    "factoryName": {
      "type": "string",
      "metadata": {
        "description": "Data Factory name"
      }
    },
    "databricksServiceName": {
      "type": "string",
      "metadata": {
        "description": "Name of the Azure Databricks linked service"
      }
    },
    "databricksWorkspaceUrl": {
      "type": "string",
      "metadata": {
        "description": "URL of the Databricks Workspace"
      }
    },
    "databricksWorkspaceName": {
      "type": "string",
      "metadata": {
        "description": "Name of the Databricks Workspace"
      }
    },
    "databricksPoolId": {
      "type": "string",
      "metadata": {
        "description": "ID of the Databricks Instance Pool"
      }
    },
    "databricksClusterNumWorkers": {
      "type": "string",
      "metadata": {
        "description": "Number of worker nodes of the Databricks Cluster"
      }
    },
    "databricksClusterSparkVersion": {
      "type": "string",
      "metadata": {
        "description": "Azure Databricks Cluster Spark Version"
      }
    },
    "databricksClusterLogPath": {
      "type": "string",
      "defaultValue": "dbfs:/cluster-logs",
      "metadata": {
        "description": "Location to deliver Spark driver, worker, and event logs"
      }
    },
    "dataServicePrincipalClientId": {
      "type": "string",
      "metadata": {
        "description": "Data Pipeline Service Principal Client Id"
      }
    },
    "databricksSecretScopeName":{
      "type": "string",
      "metadata": {
        "description": "Name of the Databricks Secret Scope"
      }
    },
    "secretNameClientSecret": {
      "type": "string",
      "metadata": {
        "description": "Name of the secret that holds the data pipeline Service Principal Client Secret"
      }
    }
  },
  "variables": {
    "databricksWorkspaceId": "[resourceId('Microsoft.Databricks/workspaces', parameters('databricksWorkspaceName'))]"
  },
  "resources": [
    {
      "name": "[concat(parameters('factoryName'), '/', parameters('databricksServiceName'))]",
      "type": "Microsoft.DataFactory/factories/linkedServices",
      "apiVersion": "2018-06-01",
      "properties": {
        "annotations": [],
        "type": "AzureDatabricks",
        "typeProperties": {
          "domain": "[parameters('databricksWorkspaceUrl')]",
          "authentication": "MSI",
          "workspaceResourceId": "[variables('databricksWorkspaceId')]",
          "instancePoolId": "[parameters('databricksPoolId')]",
          "newClusterNumOfWorker": "[parameters('databricksClusterNumWorkers')]",
          "newClusterCustomTags": {
            "DataFactoryName": "[parameters('factoryName')]"
          },
          "newClusterSparkConf": {
            "fs.azure.account.auth.type": "OAuth",
            "fs.azure.account.oauth.provider.type": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",
            "fs.azure.account.oauth2.client.id": "[parameters('dataServicePrincipalClientId')]",
            "fs.azure.account.oauth2.client.secret": "[concat('{{secrets/', parameters('databricksSecretScopeName'), '/', parameters('secretNameClientSecret'), '}}')]",
            "fs.azure.account.oauth2.client.endpoint": "[concat('https://login.microsoftonline.com/', subscription().tenantId, '/oauth2/token')]"
          },
          "newClusterLogDestination": "[parameters('databricksClusterLogPath')]",
          "newClusterSparkEnvVars": {
            "PYSPARK_PYTHON": "/databricks/python3/bin/python3"
          },
          "newClusterVersion": "[parameters('databricksClusterSparkVersion')]"
        }
      },
      "dependsOn": []
    }
  ],
  "outputs": {
    "databricksServiceName": {
      "type": "string",
      "value": "[parameters('databricksServiceName')]"
    }
  }
}