Compute
Overview
Compute APIs let you search Databricks workloads and surface inefficient applications so that you can debug, optimize, and prioritize remediation work. They provide list views and targeted inefficient‑apps views, with rich metadata for each workload and aggregations for event types and entities.
The Compute APIs include:
List API: Paginates through Spark applications, with filters for status, type, time range, and sorting, and returns both results and summary metadata.
Inefficient Events API: Aggregates counts of inefficiency event types (for example, ContendedDriverEvent, RightSizingEvent) over a selected time range and entity types.
Inefficient Apps API: Lists the individual applications that triggered selected inefficient events, with full app metadata and aggregations for event type, entity, cluster type, queue, and user.
List API
Request
POST /api/v1/apps/unifiedsearch
Use this API to search and list applications (for example, Spark workloads), filtered by status, type, and time window, with pagination and sorting support.
Body Parameters
Name | Type | Example value | Description |
|---|---|---|---|
appStatus | array[string] | ["K","F","R","S","P","U","W"] | List of application statuses to include in the search. |
appTypes | array[string] | ["spark"] | Application types to filter (for example, spark). |
start_time | string (ISO 8601) | 2025-10-29T17:13:46+05:30 | Start of the time range for the search. |
end_time | string (ISO 8601) | 2025-11-05T17:13:46+05:30 | End of the time range for the search. |
from | integer | 0 | Pagination offset (starting record). |
sort | array[object] | [{"startTime":{"order":"desc"}}] | Sort field and order. |
size | string | integer | "18" | Number of records to return per page. |
queryOnFinishedTime | boolean | false | Whether to query on finished time instead of start time. |
Examples
curl -X POST \ -H "Authorization: JWT <token>" \ -H "Content-Type: application/json" \ -H "Accept: application/json" \ -d '{ "appStatus": ["K", "F", "R", "S", "P", "U", "W"], "appTypes": ["spark"], "start_time": "2025-10-29T17:13:46+05:30", "end_time": "2025-11-05T17:13:46+05:30", "from": 0, "sort": [ { "startTime": { "order": "desc" } } ], "size": "18", "queryOnFinishedTime": false }' \ "http://23.21.77.89:3000/api/v1/apps/unifiedsearch"{"metadata":{"duration":{"max":4203185,"min":613320},"cost":{"max":3.623613290786743,"min":0.45415522050857543},"resource":{"max":null,"min":null},"events":{"max":1,"min":0},"totalEvents":4,"totalReads":11939572,"totalWrites":19834809,"appTypes":{"spark":5},"appStatus":{"S":5},"users":{"spattnaik@unraveldata.com":5},"queues":{"dnarkhede":5},"clusters":{"1013-045853-8hd9802o":1,"1104-061651-8t003h8o":1,"1104-063934-osqmqq9r":1,"1104-065536-6naljxhy":1,"1104-070852-rvlat5bz":1},"totalRecords":5,"clusterTypes":{"AUTOMATED":4,"INTERACTIVE":1}},"results":[{"appId":"app-20251104071007-0000-7280288211151368346dbx","appType":"wfi","appt":"wfi","gotoId":"app-20251104071007-0000-7280288211151368346dbx","gotoLevel":"WFI","id":"app-20251104071007-0000","nick":"spark","name":"job-151669598391028-run-953826205550441-Job_cluster","queue":"dnarkhede","status":"S","status_long":"Success","user":"spattnaik@un ...","raw_user":"spattnaik@unraveldata.com","realUser":"spattnaik@un ...","raw_realUser":"spattnaik@unraveldata.com","submitHost":"-","remarks":["AA2","IN"],"aa2Badge":true,"inBadge":true,"key":"YARN","cacheHit":false,"parentId":null,"queryStringFull":null,"clusterId":"job-151669598391028-run-953826205550441-Job_cluster","clusterUid":"1104-070852-rvlat5bz","clusterTag":"adb-1764446076781711.11.azuredatabricks.net","start_time":"11/04/25 07:10:00","start_time_long":"2025-11-04T07:10:00.693Z","finished_time":"11/04/25 07:20:14","finished_time_long":"2025-11-04T07:20:14.013Z","duration_long":613320,"predDuration_long":0,"io_long":1375160,"read_long":1015155,"write_long":360005,"resource":0,"service":0,"events":1,"numApps":0,"numSparkApps":19,"numMRJobs":19,"numEvents":1,"mrJobIds":[],"appIds":[],"sm":73,"sr":null,"fm":null,"fr":null,"km":null,"kr":null,"ss":73,"fs":null,"ks":null,"fsu":null,"ksu":null,"totalMapTasks":73,"totalReduceTasks":null,"totalSparkTasks":73,"totalMapSlotDuration":134120,"totalReduceSlotDuration":null,"totalSparkSlotDuration":134120,"inputTables":null,"outputTables":null,"wi":"app-20251104071007-0000-7280288211151368346dbx","wn":"151669598391028_1764446076781711","wt":"app-20251104071007-0000","type":"REGULAR","type_regular":true,"jobId":151669598391028,"runName":"streaming_consumer_1","runId":953826205550441,"dbus":1.9166248,"dbuCost":0.5749874,"cents":0.14719679951667786,"cost":0.7221841995166779,"dbuPremium":"-","dbuStandard":"-","clusterType":"AUTOMATED","setupDuration":0,"cleanupDuration":0,"wsInstance":"adb-1764446076781711.11.azuredatabricks.net","clusterTg":"adb-1764446076781711.11.azuredatabricks.net","sparkAppId":"-","metrics":null,"totalProcessingTime":0,"memorySeconds":0,"cpuTime":0,"storageWaitTime":0,"networkSendWaitTime":0,"networkReceiveWaitTime":0,"elastic":true,"kind":"spark","kindLong":"Spark","name_long":"job-151669598391028-run-953826205550441-Job_cluster","goToApp":{"appId":"app-20251104071007-0000-7280288211151368346dbx","appType":"WFI"},"kind_url":"spark","kind_parent_url":"workflow"},{"appId":"app-20251104065811-0000-7280265403909999102dbx","appType":"wfi","appt":"wfi","gotoId":"app-20251104065811-0000-7280265403909999102dbx","gotoLevel":"WFI","id":"app-20251104065811-0000","nick":"spark","name":"job-151669598391028-run-347433663084957-Job_cluster","queue":"dnarkhede","status":"S","status_long":"Success","user":"spattnaik@un ...","raw_user":"spattnaik@unraveldata.com","realUser":"spattnaik@un ...","raw_realUser":"spattnaik@unraveldata.com","submitHost":"-","remarks":["AA2","IN"],"aa2Badge":true,"inBadge":true,"key":"YARN","cacheHit":false,"parentId":null,"queryStringFull":null,"clusterId":"job-151669598391028-run-347433663084957-Job_cluster","clusterUid":"1104-065536-6naljxhy","clusterTag":"adb-1764446076781711.11.azuredatabricks.net","start_time":"11/04/25 06:58:04","start_time_long":"2025-11-04T06:58:04.696Z","finished_time":"11/04/25 07:08:49","finished_time_long":"2025-11-04T07:08:49.438Z","duration_long":644742,"predDuration_long":0,"io_long":2131432,"read_long":1403598,"write_long":727834,"resource":0,"service":0,"events":1,"numApps":0,"numSparkApps":26,"numMRJobs":26,"numEvents":1,"mrJobIds":[],"appIds":[],"sm":180,"sr":null,"fm":null,"fr":null,"km":null,"kr":null,"ss":180,"fs":null,"ks":null,"fsu":null,"ksu":null,"totalMapTasks":180,"totalReduceTasks":null,"totalSparkTasks":180,"totalMapSlotDuration":319935,"totalReduceSlotDuration":null,"totalSparkSlotDuration":319935,"inputTables":null,"outputTables":null,"wi":"app-20251104065811-0000-7280265403909999102dbx","wn":"151669598391028_1764446076781711","wt":"app-20251104065811-0000","type":"REGULAR","type_regular":true,"jobId":151669598391028,"runName":"streaming_consumer_1","runId":347433663084957,"dbus":3.3580313,"dbuCost":1.0074092,"cents":0.2578967809677124,"cost":1.2653059809677123,"dbuPremium":"-","dbuStandard":"-","clusterType":"AUTOMATED","setupDuration":0,"cleanupDuration":0,"wsInstance":"adb-1764446076781711.11.azuredatabricks.net","clusterTg":"adb-1764446076781711.11.azuredatabricks.net","sparkAppId":"-","metrics":null,"totalProcessingTime":0,"memorySeconds":0,"cpuTime":0,"storageWaitTime":0,"networkSendWaitTime":0,"networkReceiveWaitTime":0,"elastic":true,"kind":"spark","kindLong":"Spark","name_long":"job-151669598391028-run-347433663084957-Job_cluster","goToApp":{"appId":"app-20251104065811-0000-7280265403909999102dbx","appType":"WFI"},"kind_url":"spark","kind_parent_url":"workflow"},{"appId":"app-20251104064332-0000-7280264415259527619dbx","appType":"wfi","appt":"wfi","gotoId":"app-20251104064332-0000-7280264415259527619dbx","gotoLevel":"WFI","id":"app-20251104064332-0000","nick":"spark","name":"job-151669598391028-run-37519736376967-Job_cluster","queue":"dnarkhede","status":"S","status_long":"Success","user":"spattnaik@un ...","raw_user":"spattnaik@unraveldata.com","realUser":"spattnaik@un ...","raw_realUser":"spattnaik@unraveldata.com","submitHost":"-","remarks":["AA2","IN"],"aa2Badge":true,"inBadge":true,"key":"YARN","cacheHit":false,"parentId":null,"queryStringFull":null,"clusterId":"job-151669598391028-run-37519736376967-Job_cluster","clusterUid":"1104-063934-osqmqq9r","clusterTag":"adb-1764446076781711.11.azuredatabricks.net","start_time":"11/04/25 06:43:26","start_time_long":"2025-11-04T06:43:26.287Z","finished_time":"11/04/25 06:55:30","finished_time_long":"2025-11-04T06:55:30.759Z","duration_long":724472,"predDuration_long":0,"io_long":5535150,"read_long":5120659,"write_long":414491,"resource":0,"service":0,"events":1,"numApps":0,"numSparkApps":42,"numMRJobs":42,"numEvents":1,"mrJobIds":[],"appIds":[],"sm":265,"sr":null,"fm":null,"fr":null,"km":null,"kr":null,"ss":265,"fs":null,"ks":null,"fsu":null,"ksu":null,"totalMapTasks":265,"totalReduceTasks":null,"totalSparkTasks":265,"totalMapSlotDuration":320226,"totalReduceSlotDuration":null,"totalSparkSlotDuration":320226,"inputTables":null,"outputTables":null,"wi":"app-20251104064332-0000-7280264415259527619dbx","wn":"151669598391028_1764446076781711","wt":"app-20251104064332-0000","type":"REGULAR","type_regular":true,"jobId":151669598391028,"runName":"streaming_consumer_1","runId":37519736376967,"dbus":1.5093167,"dbuCost":0.452795,"cents":0.2897888123989105,"cost":0.7425838123989106,"dbuPremium":"-","dbuStandard":"-","clusterType":"AUTOMATED","setupDuration":0,"cleanupDuration":0,"wsInstance":"adb-1764446076781711.11.azuredatabricks.net","clusterTg":"adb-1764446076781711.11.azuredatabricks.net","sparkAppId":"-","metrics":null,"totalProcessingTime":0,"memorySeconds":0,"cpuTime":0,"storageWaitTime":0,"networkSendWaitTime":0,"networkReceiveWaitTime":0,"elastic":true,"kind":"spark","kindLong":"Spark","name_long":"job-151669598391028-run-37519736376967-Job_cluster","goToApp":{"appId":"app-20251104064332-0000-7280264415259527619dbx","appType":"WFI"},"kind_url":"spark","kind_parent_url":"workflow"},{"appId":"local-1762237252978-6368152939393537653dbx","appType":"wfi","appt":"wfi","gotoId":"local-1762237252978-6368152939393537653dbx","gotoLevel":"WFI","id":"local-1762237252978","nick":"spark","name":"Siddhanta's Cluster 2025-10-13 10:28:39","queue":"dnarkhede","status":"S","status_long":"Success","user":"spattnaik@un ...","raw_user":"spattnaik@unraveldata.com","realUser":"spattnaik@un ...","raw_realUser":"spattnaik@unraveldata.com","submitHost":"-","remarks":["AA2"],"aa2Badge":true,"inBadge":false,"key":"YARN","cacheHit":false,"parentId":null,"queryStringFull":null,"clusterId":"Siddhanta's Cluster 2025-10-13 10:28:39","clusterUid":"1013-045853-8hd9802o","clusterTag":"adb-1764446076781711.11.azuredatabricks.net","start_time":"11/04/25 06:20:49","start_time_long":"2025-11-04T06:20:49.557Z","finished_time":"11/04/25 07:30:52","finished_time_long":"2025-11-04T07:30:52.742Z","duration_long":4203185,"predDuration_long":0,"io_long":18611500,"read_long":442539,"write_long":18168961,"resource":0,"service":0,"events":0,"numApps":0,"numSparkApps":227,"numMRJobs":227,"numEvents":0,"mrJobIds":[],"appIds":[],"sm":1426,"sr":null,"fm":null,"fr":null,"km":null,"kr":null,"ss":1426,"fs":null,"ks":null,"fsu":null,"ksu":null,"totalMapTasks":1427,"totalReduceTasks":null,"totalSparkTasks":1427,"totalMapSlotDuration":245655,"totalReduceSlotDuration":null,"totalSparkSlotDuration":245655,"inputTables":null,"outputTables":null,"wi":"local-1762237252978-6368152939393537653dbx","wn":"1013-045853-8hd9802o_1764446076781711","wt":"local-1762237252978","type":"REGULAR","type_regular":true,"jobId":"-","runName":"-","runId":"-","dbus":4.670206,"dbuCost":2.5686133,"cents":1.0554664134979248,"cost":3.624079713497925,"dbuPremium":"-","dbuStandard":"-","clusterType":"INTERACTIVE","setupDuration":0,"cleanupDuration":0,"wsInstance":"adb-1764446076781711.11.azuredatabricks.net","clusterTg":"adb-1764446076781711.11.azuredatabricks.net","sparkAppId":"-","metrics":null,"totalProcessingTime":0,"memorySeconds":0,"cpuTime":0,"storageWaitTime":0,"networkSendWaitTime":0,"networkReceiveWaitTime":0,"elastic":true,"kind":"spark","kindLong":"Spark","name_long":"Siddhanta's Cluster 2025-10-13 10:28:39","goToApp":{"appId":"local-1762237252978-6368152939393537653dbx","appType":"WFI"},"kind_url":"spark","kind_parent_url":"workflow"},{"appId":"app-20251104062047-0000-7280262627970259840dbx","appType":"wfi","appt":"wfi","gotoId":"app-20251104062047-0000-7280262627970259840dbx","gotoLevel":"WFI","id":"app-20251104062047-0000","nick":"spark","name":"job-151669598391028-run-406883658197206-Job_cluster","queue":"dnarkhede","status":"S","status_long":"Success","user":"spattnaik@un ...","raw_user":"spattnaik@unraveldata.com","realUser":"spattnaik@un ...","raw_realUser":"spattnaik@unraveldata.com","submitHost":"-","remarks":["AA2","IN"],"aa2Badge":true,"inBadge":true,"key":"YARN","cacheHit":false,"parentId":null,"queryStringFull":null,"clusterId":"job-151669598391028-run-406883658197206-Job_cluster","clusterUid":"1104-061651-8t003h8o","clusterTag":"adb-1764446076781711.11.azuredatabricks.net","start_time":"11/04/25 06:20:40","start_time_long":"2025-11-04T06:20:40.385Z","finished_time":"11/04/25 06:39:09","finished_time_long":"2025-11-04T06:39:09.006Z","duration_long":1108621,"predDuration_long":0,"io_long":4121139,"read_long":3957621,"write_long":163518,"resource":0,"service":0,"events":1,"numApps":0,"numSparkApps":27,"numMRJobs":27,"numEvents":1,"mrJobIds":[],"appIds":[],"sm":66,"sr":null,"fm":null,"fr":null,"km":null,"kr":null,"ss":66,"fs":null,"ks":null,"fsu":null,"ksu":null,"totalMapTasks":66,"totalReduceTasks":null,"totalSparkTasks":66,"totalMapSlotDuration":62922,"totalReduceSlotDuration":null,"totalSparkSlotDuration":62922,"inputTables":null,"outputTables":null,"wi":"app-20251104062047-0000-7280262627970259840dbx","wn":"151669598391028_1764446076781711","wt":"app-20251104062047-0000","type":"REGULAR","type_regular":true,"jobId":151669598391028,"runName":"streaming_consumer_1","runId":406883658197206,"dbus":0.92385095,"dbuCost":0.27715522,"cents":0.17737935483455658,"cost":0.45453457483455656,"dbuPremium":"-","dbuStandard":"-","clusterType":"AUTOMATED","setupDuration":0,"cleanupDuration":0,"wsInstance":"adb-1764446076781711.11.azuredatabricks.net","clusterTg":"adb-1764446076781711.11.azuredatabricks.net","sparkAppId":"-","metrics":null,"totalProcessingTime":0,"memorySeconds":0,"cpuTime":0,"storageWaitTime":0,"networkSendWaitTime":0,"networkReceiveWaitTime":0,"elastic":true,"kind":"spark","kindLong":"Spark","name_long":"job-151669598391028-run-406883658197206-Job_cluster","goToApp":{"appId":"app-20251104062047-0000-7280262627970259840dbx","appType":"WFI"},"kind_url":"spark","kind_parent_url":"workflow"}]}Inefficient Events API
GET /api/v1/apps/events/inefficient_apps
Use this API to retrieve aggregate counts of inefficiency event types for applications within a specified time range and entity set.
Query Parameters
Name | Type | Example value | Description |
|---|---|---|---|
start_time | string (ISO 8601) | 2025-11-04T00:00:00+05:30 | Start timestamp for filtering inefficient applications. |
end_time | string (ISO 8601) | 2025-11-04T23:59:59+05:30 | End timestamp for filtering inefficient applications. |
entity | array[integer] | [2,6] | Entity identifiers to include (for example, job run or cluster). |
Example
curl -X GET \ -H "Authorization: JWT <token>" \ -H "Content-Type: application/json" \ -H "Accept: application/json" \ "yourdomain.com/api/v1/apps/events/inefficient_apps?start_time=2025-11-04T00:00:00+05:30&end_time=2025-11-04T23:59:59+05:30&entity=[2,6]"
[
{
"key": "ContendedDriverEvent",
"doc_count": 5,
"distinct_terms": { "value": 5 }
},
{
"key": "AutoScaleConfigEvent",
"doc_count": 3,
"distinct_terms": { "value": 3 }
},
{
"key": "RightSizingEvent",
"doc_count": 2,
"distinct_terms": { "value": 2 }
}
]Inefficient Apps API
Request
GET /api/v1/apps/events/inefficient_apps_newux/
Use this API to list individual inefficient applications that match selected event types and entity types over a given time range, with pagination and sorting.
Query Parameters
Name | Type | Example value | Description |
|---|---|---|---|
start_time | string (ISO 8601) | 2025-11-04T00:00:00+05:30 | Start of the time range to fetch inefficient apps. |
end_time | string (ISO 8601) | 2025-11-04T23:59:59+05:30 | End of the time range. |
entityType | array[integer] | [2,6] | Entity types such as job run or cluster. |
eventName | array[string] | ["ContendedDriverEvent","AutoScaleConfigEvent","RightSizingEvent","RightSizingWithPhotonBenefitEvent","SingleNodeClusterEvent"] | Inefficiency event names to filter on. |
sort | string | startTime | Sort field (usually startTime). |
from | integer | 0 | Pagination offset (starting index). |
size | integer | 15 | Number of records to return per page. |
Example
curl -X GET \ -H "Authorization: JWT <token>" \ -H "Accept: application/json" \ "http://23.21.77.89:3000/api/v1/apps/events/inefficient_apps_newux/?start_time=2025-11-04T00:00:00+05:30&end_time=2025-11-04T23:59:59+05:30&entityType=[2,6]&eventName=[\"ContendedDriverEvent\",\"AutoScaleConfigEvent\",\"RightSizingEvent\",\"RightSizingWithPhotonBenefitEvent\",\"SingleNodeClusterEvent\"]&sort=startTime&from=0&size=15"
{"results":[{"appId":"app-20251104071007-0000-7280288211151368346dbx","appType":"wfi","appt":"wfi","gotoId":"app-20251104071007-0000-7280288211151368346dbx","gotoLevel":"WFI","id":"app-20251104071007-0000","nick":"spark","name":"job-151669598391028-run-953826205550441-Job_cluster","queue":"dnarkhede","status":"S","status_long":"Success","user":"spattnaik@un ...","raw_user":"spattnaik@unraveldata.com","realUser":"spattnaik@un ...","raw_realUser":"spattnaik@unraveldata.com","submitHost":"-","remarks":["AA2","IN"],"aa2Badge":true,"inBadge":true,"key":"YARN","cacheHit":false,"parentId":null,"queryStringFull":null,"clusterId":"job-151669598391028-run-953826205550441-Job_cluster","clusterUid":"1104-070852-rvlat5bz","clusterTag":"adb-1764446076781711.11.azuredatabricks.net","start_time":"11/04/25 07:10:00","start_time_long":"2025-11-04T07:10:00.693Z","finished_time":"11/04/25 07:20:14","finished_time_long":"2025-11-04T07:20:14.013Z","duration_long":613320,"predDuration_long":0,"io_long":1375160,"read_long":1015155,"write_long":360005,"resource":0,"service":0,"events":1,"numApps":0,"numSparkApps":19,"numMRJobs":19,"numEvents":1,"mrJobIds":[],"appIds":[],"sm":73,"sr":null,"fm":null,"fr":null,"km":null,"kr":null,"ss":73,"fs":null,"ks":null,"fsu":null,"ksu":null,"totalMapTasks":73,"totalReduceTasks":null,"totalSparkTasks":73,"totalMapSlotDuration":134120,"totalReduceSlotDuration":null,"totalSparkSlotDuration":134120,"inputTables":null,"outputTables":null,"wi":"app-20251104071007-0000-7280288211151368346dbx","wn":"151669598391028_1764446076781711","wt":"app-20251104071007-0000","type":"REGULAR","type_regular":true,"jobId":151669598391028,"runName":"streaming_consumer_1","runId":953826205550441,"dbus":1.9166248,"dbuCost":0.5749874,"cents":0.14719679951667786,"cost":0.7221841995166779,"dbuPremium":"-","dbuStandard":"-","clusterType":"AUTOMATED","setupDuration":0,"cleanupDuration":0,"wsInstance":"adb-1764446076781711.11.azuredatabricks.net","clusterTg":"adb-1764446076781711.11.azuredatabricks.net","sparkAppId":"-","metrics":null,"totalProcessingTime":0,"memorySeconds":0,"cpuTime":0,"storageWaitTime":0,"networkSendWaitTime":0,"networkReceiveWaitTime":0,"elastic":true,"kind":"spark","kindLong":"Spark","name_long":"job-151669598391028-run-953826205550441-Job_cluster","goToApp":{"appId":"app-20251104071007-0000-7280288211151368346dbx","appType":"WFI"},"kind_url":"spark","kind_parent_url":"workflow"},{"appId":"app-20251104065811-0000-7280265403909999102dbx","appType":"wfi","appt":"wfi","gotoId":"app-20251104065811-0000-7280265403909999102dbx","gotoLevel":"WFI","id":"app-20251104065811-0000","nick":"spark","name":"job-151669598391028-run-347433663084957-Job_cluster","queue":"dnarkhede","status":"S","status_long":"Success","user":"spattnaik@un ...","raw_user":"spattnaik@unraveldata.com","realUser":"spattnaik@un ...","raw_realUser":"spattnaik@unraveldata.com","submitHost":"-","remarks":["AA2","IN"],"aa2Badge":true,"inBadge":true,"key":"YARN","cacheHit":false,"parentId":null,"queryStringFull":null,"clusterId":"job-151669598391028-run-347433663084957-Job_cluster","clusterUid":"1104-065536-6naljxhy","clusterTag":"adb-1764446076781711.11.azuredatabricks.net","start_time":"11/04/25 06:58:04","start_time_long":"2025-11-04T06:58:04.696Z","finished_time":"11/04/25 07:08:49","finished_time_long":"2025-11-04T07:08:49.438Z","duration_long":644742,"predDuration_long":0,"io_long":2131432,"read_long":1403598,"write_long":727834,"resource":0,"service":0,"events":1,"numApps":0,"numSparkApps":26,"numMRJobs":26,"numEvents":1,"mrJobIds":[],"appIds":[],"sm":180,"sr":null,"fm":null,"fr":null,"km":null,"kr":null,"ss":180,"fs":null,"ks":null,"fsu":null,"ksu":null,"totalMapTasks":180,"totalReduceTasks":null,"totalSparkTasks":180,"totalMapSlotDuration":319935,"totalReduceSlotDuration":null,"totalSparkSlotDuration":319935,"inputTables":null,"outputTables":null,"wi":"app-20251104065811-0000-7280265403909999102dbx","wn":"151669598391028_1764446076781711","wt":"app-20251104065811-0000","type":"REGULAR","type_regular":true,"jobId":151669598391028,"runName":"streaming_consumer_1","runId":347433663084957,"dbus":3.3580313,"dbuCost":1.0074092,"cents":0.2578967809677124,"cost":1.2653059809677123,"dbuPremium":"-","dbuStandard":"-","clusterType":"AUTOMATED","setupDuration":0,"cleanupDuration":0,"wsInstance":"adb-1764446076781711.11.azuredatabricks.net","clusterTg":"adb-1764446076781711.11.azuredatabricks.net","sparkAppId":"-","metrics":null,"totalProcessingTime":0,"memorySeconds":0,"cpuTime":0,"storageWaitTime":0,"networkSendWaitTime":0,"networkReceiveWaitTime":0,"elastic":true,"kind":"spark","kindLong":"Spark","name_long":"job-151669598391028-run-347433663084957-Job_cluster","goToApp":{"appId":"app-20251104065811-0000-7280265403909999102dbx","appType":"WFI"},"kind_url":"spark","kind_parent_url":"workflow"},{"appId":"app-20251104064332-0000-7280264415259527619dbx","appType":"wfi","appt":"wfi","gotoId":"app-20251104064332-0000-7280264415259527619dbx","gotoLevel":"WFI","id":"app-20251104064332-0000","nick":"spark","name":"job-151669598391028-run-37519736376967-Job_cluster","queue":"dnarkhede","status":"S","status_long":"Success","user":"spattnaik@un ...","raw_user":"spattnaik@unraveldata.com","realUser":"spattnaik@un ...","raw_realUser":"spattnaik@unraveldata.com","submitHost":"-","remarks":["AA2","IN"],"aa2Badge":true,"inBadge":true,"key":"YARN","cacheHit":false,"parentId":null,"queryStringFull":null,"clusterId":"job-151669598391028-run-37519736376967-Job_cluster","clusterUid":"1104-063934-osqmqq9r","clusterTag":"adb-1764446076781711.11.azuredatabricks.net","start_time":"11/04/25 06:43:26","start_time_long":"2025-11-04T06:43:26.287Z","finished_time":"11/04/25 06:55:30","finished_time_long":"2025-11-04T06:55:30.759Z","duration_long":724472,"predDuration_long":0,"io_long":5535150,"read_long":5120659,"write_long":414491,"resource":0,"service":0,"events":1,"numApps":0,"numSparkApps":42,"numMRJobs":42,"numEvents":1,"mrJobIds":[],"appIds":[],"sm":265,"sr":null,"fm":null,"fr":null,"km":null,"kr":null,"ss":265,"fs":null,"ks":null,"fsu":null,"ksu":null,"totalMapTasks":265,"totalReduceTasks":null,"totalSparkTasks":265,"totalMapSlotDuration":320226,"totalReduceSlotDuration":null,"totalSparkSlotDuration":320226,"inputTables":null,"outputTables":null,"wi":"app-20251104064332-0000-7280264415259527619dbx","wn":"151669598391028_1764446076781711","wt":"app-20251104064332-0000","type":"REGULAR","type_regular":true,"jobId":151669598391028,"runName":"streaming_consumer_1","runId":37519736376967,"dbus":1.5093167,"dbuCost":0.452795,"cents":0.2897888123989105,"cost":0.7425838123989106,"dbuPremium":"-","dbuStandard":"-","clusterType":"AUTOMATED","setupDuration":0,"cleanupDuration":0,"wsInstance":"adb-1764446076781711.11.azuredatabricks.net","clusterTg":"adb-1764446076781711.11.azuredatabricks.net","sparkAppId":"-","metrics":null,"totalProcessingTime":0,"memorySeconds":0,"cpuTime":0,"storageWaitTime":0,"networkSendWaitTime":0,"networkReceiveWaitTime":0,"elastic":true,"kind":"spark","kindLong":"Spark","name_long":"job-151669598391028-run-37519736376967-Job_cluster","goToApp":{"appId":"app-20251104064332-0000-7280264415259527619dbx","appType":"WFI"},"kind_url":"spark","kind_parent_url":"workflow"},{"appId":"local-1762237252978-6368152939393537653dbx","appType":"wfi","appt":"wfi","gotoId":"local-1762237252978-6368152939393537653dbx","gotoLevel":"WFI","id":"local-1762237252978","nick":"spark","name":"Siddhanta's Cluster 2025-10-13 10:28:39","queue":"dnarkhede","status":"S","status_long":"Success","user":"spattnaik@un ...","raw_user":"spattnaik@unraveldata.com","realUser":"spattnaik@un ...","raw_realUser":"spattnaik@unraveldata.com","submitHost":"-","remarks":["AA2"],"aa2Badge":true,"inBadge":false,"key":"YARN","cacheHit":false,"parentId":null,"queryStringFull":null,"clusterId":"Siddhanta's Cluster 2025-10-13 10:28:39","clusterUid":"1013-045853-8hd9802o","clusterTag":"adb-1764446076781711.11.azuredatabricks.net","start_time":"11/04/25 06:20:49","start_time_long":"2025-11-04T06:20:49.557Z","finished_time":"11/04/25 07:30:52","finished_time_long":"2025-11-04T07:30:52.742Z","duration_long":4203185,"predDuration_long":0,"io_long":18611500,"read_long":442539,"write_long":18168961,"resource":0,"service":0,"events":0,"numApps":0,"numSparkApps":227,"numMRJobs":227,"numEvents":0,"mrJobIds":[],"appIds":[],"sm":1426,"sr":null,"fm":null,"fr":null,"km":null,"kr":null,"ss":1426,"fs":null,"ks":null,"fsu":null,"ksu":null,"totalMapTasks":1427,"totalReduceTasks":null,"totalSparkTasks":1427,"totalMapSlotDuration":245655,"totalReduceSlotDuration":null,"totalSparkSlotDuration":245655,"inputTables":null,"outputTables":null,"wi":"local-1762237252978-6368152939393537653dbx","wn":"1013-045853-8hd9802o_1764446076781711","wt":"local-1762237252978","type":"REGULAR","type_regular":true,"jobId":"-","runName":"-","runId":"-","dbus":4.670206,"dbuCost":2.5686133,"cents":1.0554664134979248,"cost":3.624079713497925,"dbuPremium":"-","dbuStandard":"-","clusterType":"INTERACTIVE","setupDuration":0,"cleanupDuration":0,"wsInstance":"adb-1764446076781711.11.azuredatabricks.net","clusterTg":"adb-1764446076781711.11.azuredatabricks.net","sparkAppId":"-","metrics":null,"totalProcessingTime":0,"memorySeconds":0,"cpuTime":0,"storageWaitTime":0,"networkSendWaitTime":0,"networkReceiveWaitTime":0,"elastic":true,"kind":"spark","kindLong":"Spark","name_long":"Siddhanta's Cluster 2025-10-13 10:28:39","goToApp":{"appId":"local-1762237252978-6368152939393537653dbx","appType":"WFI"},"kind_url":"spark","kind_parent_url":"workflow"},{"appId":"app-20251104062047-0000-7280262627970259840dbx","appType":"wfi","appt":"wfi","gotoId":"app-20251104062047-0000-7280262627970259840dbx","gotoLevel":"WFI","id":"app-20251104062047-0000","nick":"spark","name":"job-151669598391028-run-406883658197206-Job_cluster","queue":"dnarkhede","status":"S","status_long":"Success","user":"spattnaik@un ...","raw_user":"spattnaik@unraveldata.com","realUser":"spattnaik@un ...","raw_realUser":"spattnaik@unraveldata.com","submitHost":"-","remarks":["AA2","IN"],"aa2Badge":true,"inBadge":true,"key":"YARN","cacheHit":false,"parentId":null,"queryStringFull":null,"clusterId":"job-151669598391028-run-406883658197206-Job_cluster","clusterUid":"1104-061651-8t003h8o","clusterTag":"adb-1764446076781711.11.azuredatabricks.net","start_time":"11/04/25 06:20:40","start_time_long":"2025-11-04T06:20:40.385Z","finished_time":"11/04/25 06:39:09","finished_time_long":"2025-11-04T06:39:09.006Z","duration_long":1108621,"predDuration_long":0,"io_long":4121139,"read_long":3957621,"write_long":163518,"resource":0,"service":0,"events":1,"numApps":0,"numSparkApps":27,"numMRJobs":27,"numEvents":1,"mrJobIds":[],"appIds":[],"sm":66,"sr":null,"fm":null,"fr":null,"km":null,"kr":null,"ss":66,"fs":null,"ks":null,"fsu":null,"ksu":null,"totalMapTasks":66,"totalReduceTasks":null,"totalSparkTasks":66,"totalMapSlotDuration":62922,"totalReduceSlotDuration":null,"totalSparkSlotDuration":62922,"inputTables":null,"outputTables":null,"wi":"app-20251104062047-0000-7280262627970259840dbx","wn":"151669598391028_1764446076781711","wt":"app-20251104062047-0000","type":"REGULAR","type_regular":true,"jobId":151669598391028,"runName":"streaming_consumer_1","runId":406883658197206,"dbus":0.92385095,"dbuCost":0.27715522,"cents":0.17737935483455658,"cost":0.45453457483455656,"dbuPremium":"-","dbuStandard":"-","clusterType":"AUTOMATED","setupDuration":0,"cleanupDuration":0,"wsInstance":"adb-1764446076781711.11.azuredatabricks.net","clusterTg":"adb-1764446076781711.11.azuredatabricks.net","sparkAppId":"-","metrics":null,"totalProcessingTime":0,"memorySeconds":0,"cpuTime":0,"storageWaitTime":0,"networkSendWaitTime":0,"networkReceiveWaitTime":0,"elastic":true,"kind":"spark","kindLong":"Spark","name_long":"job-151669598391028-run-406883658197206-Job_cluster","goToApp":{"appId":"app-20251104062047-0000-7280262627970259840dbx","appType":"WFI"},"kind_url":"spark","kind_parent_url":"workflow"}],"total":5,"aggregations":{"by_type":[{"key":"ContendedDriverEvent","doc_count":5},{"key":"AutoScaleConfigEvent","doc_count":3},{"key":"RightSizingEvent","doc_count":2},{"key":"RightSizingWithPhotonBenefitEvent","doc_count":2},{"key":"SingleNodeClusterEvent","doc_count":2}],"by_entity":[{"key":2,"doc_count":14,"distinct_terms":{"value":5}}],"clusterTypes":{"AUTOMATED":4,"INTERACTIVE":1},"queues":{"dnarkhede":5},"users":{"spattnaik@unraveldata.com":5}}}