databricks-clusters-job.json

{
    "typeName": "Databricks::Clusters::Job",
    "description": "Manage Jobs running on a cluster",
    "sourceUrl": "https://github.com/aws-ia/cloudformation-databricks-resource-providers.git",
    "documentationUrl": "https://github.com/aws-ia/cloudformation-databricks-resource-providers",
    "typeConfiguration": {
        "properties": {
            "DatabricksAccess": {
                "$ref": "#/definitions/DatabricksAccess"
            }
        },
        "additionalProperties": false,
        "required": [
            "DatabricksAccess"
        ]
    },
    "definitions": {
        "DatabricksAccess": {
            "description": "Properties needed to access databricks.",
            "type": "object",
            "properties": {
                "DatabricksInstance": {
                    "type": "string",
                    "description": "Domain used to access Databricks"
                },
                "Token": {
                    "type": "string"
                }
            },
            "required": [
                "DatabricksInstance",
                "Token"
            ],
            "additionalProperties": false
        },
        "Task": {
            "type": "object",
            "properties": {
                "TaskKey": {
                    "description": "A unique name for the task. This field is used to refer to this task from other tasks. This field is required and must be unique within its parent job. On Update or Reset, this field is used to reference the tasks to be updated or reset. The maximum length is 100 characters.",
                    "type": "string"
                },
                "Description": {
                    "description": "An optional description for this task. The maximum length is 4096 bytes.",
                    "type": "string"
                },
                "DependsOn": {
                    "description": "An optional array of objects specifying the dependency graph of the task. All tasks specified in this field must complete successfully before executing this task. The key is task_key, and the value is the name assigned to the dependent task. This field is required when a job consists of more than one task.",
                    "type": "array",
                    "insertionOrder": false,
                    "items": {
                        "type": "string"
                    }
                },
                "ExistingClusterId": {
                    "description": "If existing_cluster_id, the ID of an existing cluster that is used for all runs of this task. When running tasks on an existing cluster, you may need to manually restart the cluster if it stops responding. We suggest running jobs on new clusters for greater reliability.",
                    "type": "string"
                },
                "NotebookTask": {
                    "type": "object",
                    "properties": {
                        "NotebookPath": {
                            "description": "The path of the notebook to be run in the Databricks workspace or remote repository. For notebooks stored in the Databricks workspace, the path must be absolute and begin with a slash. For notebooks stored in a remote repository, the path must be relative. This field is required.",
                            "type": "string"
                        },
                        "BaseParameters": {
                            "description": "Base parameters to be used for each run of this job. If the run is initiated by a call to run-now with parameters specified, the two parameters maps are merged. If the same key is specified in base_parameters and in run-now, the value from run-now is used.\n\nUse Task parameter variables to set parameters containing information about job runs.\n\nIf the notebook takes a parameter that is not specified in the job's base_parameters or the run-now override parameters, the default value from the notebook is used",
                            "type": "object"
                        }
                    },
                    "required": [
                        "NotebookPath"
                    ],
                    "additionalProperties": false
                },
                "SparkJarTask": {
                    "type": "object",
                    "properties": {
                        "MainClassName": {
                            "description": "The full name of the class containing the main method to be executed. This class must be contained in a JAR provided as a library.",
                            "type": "string"
                        },
                        "Parameters": {
                            "description": "Parameters passed to the main method.\n\nUse Task parameter variables to set parameters containing information about job runs.",
                            "type": "object"
                        }
                    },
                    "required": [
                        "MainClassName"
                    ],
                    "additionalProperties": false
                },
                "SparkPythonTask": {
                    "type": "object",
                    "properties": {
                        "PythonFile": {
                            "description": "The URI of the Python file to be executed. DBFS and S3 paths are supported. This field is required.",
                            "type": "string"
                        },
                        "Parameters": {
                            "description": "Command line parameters passed to the Python file.\n\nUse Task parameter variables to set parameters containing information about job runs.",
                            "type": "object"
                        }
                    },
                    "required": [
                        "PythonFile"
                    ],
                    "additionalProperties": false
                },
                "SparkSubmitTask": {
                    "type": "object",
                    "properties": {
                        "Parameters": {
                            "description": "Command-line parameters passed to spark submit.\n\nUse Task parameter variables to set parameters containing information about job runs.",
                            "type": "object"
                        }
                    },
                    "additionalProperties": false
                },
                "PipelineTask": {
                    "type": "object",
                    "properties": {
                        "PipelineId": {
                            "description": "The full name of the pipeline task to execute.",
                            "type": "string"
                        },
                        "FullRefresh": {
                            "description": "If true, a full refresh will be triggered on the delta live table.",
                            "type": "boolean"
                        }
                    },
                    "required": [
                        "PipelineId"
                    ],
                    "additionalProperties": false
                },
                "PythonWheelTask": {
                    "type": "object",
                    "properties": {
                        "PackageName": {
                            "description": "Name of the package to execute",
                            "type": "string"
                        },
                        "EntryPoint": {
                            "description": "Named entry point to use, if it does not exist in the metadata of the package it executes the function from the package directly using $packageName.$entryPoint()",
                            "type": "string"
                        },
                        "Parameters": {
                            "description": "Command-line parameters passed to Python wheel task. Leave it empty if named_parameters is not null.",
                            "type": "array",
                            "insertionOrder": false,
                            "items": {
                                "type": "string"
                            }
                        },
                        "NamedParameters": {
                            "description": "Command-line parameters passed to Python wheel task in the form of [\"--name=task\", \"--data=dbfs:/path/to/data.json\"]. Leave it empty if parameters is not null",
                            "type": "object"
                        }
                    },
                    "required": [
                        "PackageName"
                    ],
                    "additionalProperties": false
                },
                "Libraries": {
                    "description": "An optional list of libraries to be installed on the cluster that executes the task. The default value is an empty list.",
                    "type": "array",
                    "insertionOrder": false,
                    "items": {
                        "type": "object"
                    }
                },
                "EmailNotifications": {
                    "$ref": "#/definitions/EmailNotifications"
                },
                "TimeoutSeconds": {
                    "description": "An optional timeout applied to each run of this job task. The default behavior is to have no timeout.",
                    "type": "integer"
                },
                "MaxRetries": {
                    "description": "An optional maximum number of times to retry an unsuccessful run. A run is considered to be unsuccessful if it completes with the FAILED result_state or INTERNAL_ERROR life_cycle_state. The value -1 means to retry indefinitely and the value 0 means to never retry. The default behavior is to never retry.",
                    "type": "integer"
                },
                "MinRetryIntervalMillies": {
                    "description": "An optional minimal interval in milliseconds between the start of the failed run and the subsequent retry run. The default behavior is that unsuccessful runs are immediately retried.",
                    "type": "integer"
                },
                "RetryOnTimeout": {
                    "description": "An optional policy to specify whether to retry a task when it times out. The default behavior is to not retry on timeout.",
                    "type": "boolean"
                }
            },
            "required": [
                "TaskKey"
            ],
            "additionalProperties": false
        },
        "EmailNotifications": {
            "type": "object",
            "properties": {
                "OnStart": {
                    "type": "array",
                    "description": "A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.",
                    "insertionOrder": false,
                    "items": {
                        "type": "string"
                    }
                },
                "OnSuccess": {
                    "type": "array",
                    "description": "A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a TERMINATED life_cycle_state and a SUCCESSFUL result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.",
                    "insertionOrder": false,
                    "items": {
                        "type": "string"
                    }
                },
                "OnFailure": {
                    "type": "array",
                    "description": "A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an INTERNAL_ERROR life_cycle_state or a SKIPPED, FAILED, or TIMED_OUT result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent.",
                    "insertionOrder": false,
                    "items": {
                        "type": "string"
                    }
                },
                "NoAlertForSkippedRuns": {
                    "description": "If true, do not send email to recipients specified in on_failure if the run is skipped.",
                    "type": "boolean"
                }
            },
            "additionalProperties": false
        },
        "Schedule": {
            "type": "object",
            "properties": {
                "QuartzCronExpression": {
                    "description": "A Cron expression using Quartz syntax that describes the schedule for a job. See Cron Trigger for details. This field is required.\n\n",
                    "type": "string"
                },
                "TimezoneId": {
                    "description": "A Java timezone ID. The schedule for a job is resolved with respect to this timezone. See Java TimeZone for details. This field is required.\n\n",
                    "type": "string"
                },
                "PauseStatus": {
                    "description": "Indicate whether this schedule is paused or not.",
                    "type": "string",
                    "enum": [
                        "PAUSED",
                        "UNPAUSED"
                    ]
                }
            },
            "additionalProperties": false
        },
        "AccessUser": {
            "type": "object",
            "properties": {
                "UserName": {
                    "type": "string"
                },
                "PermissionLevel": {
                    "type": "string"
                }
            },
            "additionalProperties": false
        }

    },
    "properties": {
        "Name": {
            "type": "string",
            "description": "An optional name for the job."
        },
        "Tags": {
            "type": "object",
            "description": "A map of tags associated with the job. These are forwarded to the cluster as cluster tags for jobs clusters, and are subject to the same limitations as cluster tags. A maximum of 25 tags can be added to the job."
        },
        "Tasks": {
            "description": "A list of task specifications to be executed by this job.",
            "type": "array",
            "insertionOrder": false,
            "items": {
                "$ref": "#/definitions/Task"
            }
        },
        "EmailNotifications": {
            "$ref": "#/definitions/EmailNotifications"
        },
        "TimeoutSeconds": {
            "description": "An optional timeout applied to each run of this job. The default behavior is to have no timeout.",
            "type": "integer"
        },
        "Schedule": {
            "$ref": "#/definitions/Schedule"
        },
        "MaxConcurrentRuns": {
            "description": "An optional maximum allowed number of concurrent runs of the job.\n\nSet this value if you want to be able to execute multiple runs of the same job concurrently. This is useful for example if you trigger your job on a frequent schedule and want to allow consecutive runs to overlap with each other, or if you want to trigger multiple runs which differ by their input parameters.\n\nThis setting affects only new runs. For example, suppose the job's concurrency is 4 and there are 4 concurrent active runs. Then setting the concurrency to 3 won't kill any of the active runs. However, from then on, new runs are skipped unless there are fewer than 3 active runs.\n\nThis value cannot exceed 1000. Setting this value to 0 causes all new runs to be skipped. The default behavior is to allow only 1 concurrent run.",
            "type": "integer"
        },
        "Format": {
            "description": "Used to tell what is the format of the job. This field is ignored in Create/Update/Reset calls. When using the Jobs API 2.1 this value is always set to \"MULTI_TASK\".",
            "type": "string",
            "enum": [
                "SINGLE_TASK",
                "MULTI_TASK"
            ]
        },
        "AccessControlList": {
            "description": "List of permissions to set on the job.",
            "type": "array",
            "insertionOrder": false,
            "items": {
                "$ref": "#/definitions/AccessUser"
            }
        },
        "ExistingClusterId": {
            "description": "If existing_cluster_id, the ID of an existing cluster that is used for all runs of this task. When running tasks on an existing cluster, you may need to manually restart the cluster if it stops responding. We suggest running jobs on new clusters for greater reliability.",
            "type": "string"
        },
        "JobId": {
            "type": "integer"
        },
        "CreatedTime": {
            "type": "integer"
        },
        "RunAsOwner": {
            "type": "boolean"
        },
        "Settings": {
            "type": "object"
        },
        "RunAsUserName":{
            "type": "string"
        },
        "CreatorUserName": {
            "type": "string"
        }
    },
    "additionalProperties": false,
    "readOnlyProperties": [
        "/properties/JobId",
        "/properties/CreatedTime"
    ],
    "primaryIdentifier": [
        "/properties/JobId"
    ],
    "handlers": {
        "create": {
            "permissions": [
            ]
        },
        "read": {
            "permissions": [
            ]
        },
        "update": {
            "permissions": [
            ]
        },
        "delete": {
            "permissions": [
            ]
        },
        "list": {
            "permissions": [
            ]
        }
    }
}