Show the Stage ID and Task ID that corresponds to the max metric
digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: 9 ms";
2 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build: 9 ms<br>number of output rows: 1"];
}
3 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 50<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>25 ms (0 ms, 0 ms, 2 ms (stage 17.0: task 70))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 50<br>local bytes read: 2.9 KiB<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 50<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size total (min, med, max (stageId: taskId))<br>1600.0 B (32.0 B, 32.0 B, 32.0 B (stage 17.0: task 67))<br>local merged bytes read: 0.0 B<br>number of partitions: 1<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>2.9 KiB (58.0 B, 58.0 B, 68.0 B (stage 17.0: task 80))"];
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n5 ms (0 ms, 0 ms, 2 ms (stage 17.0: task 72))";
5 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>2 ms (0 ms, 0 ms, 2 ms (stage 17.0: task 72))<br>number of output rows: 50"];
6 [labelType="html" label="<br><b>Project</b><br><br>"];
7 [labelType="html" label="<b>Scan ExistingRDD Delta Table State with Stats #0 - file:/home/jovyan/spark-warehouse/test.db/fhvhv_trip/_delta_log</b><br><br>number of output rows: 4"];
}
2->0;
3->2;
5->3;
6->5;
7->6;
}
8
AdaptiveSparkPlan isFinalPlan=true
HashAggregate(keys=[], functions=[sum((_extract_numRecords#1330L - coalesce(_extract_cardinality#1331L, 0))), count(CASE WHEN isnull(_extract_numRecords#1330L) THEN 1 END), count(CASE WHEN (_extract_numRecords#1330L > 0) THEN 1 END)])
HashAggregate(keys=[], functions=[partial_sum((_extract_numRecords#1330L - coalesce(_extract_cardinality#1331L, 0))), partial_count(CASE WHEN isnull(_extract_numRecords#1330L) THEN 1 END), partial_count(CASE WHEN (_extract_numRecords#1330L > 0) THEN 1 END)])
Project [stats#1233.numRecords AS _extract_numRecords#1330L, deletionVector#1217.cardinality AS _extract_cardinality#1331L]
Scan ExistingRDD Delta Table State with Stats #0 - file:/home/jovyan/spark-warehouse/test.db/fhvhv_trip/_delta_log[path#1210,partitionValues#1211,size#1212L,modificationTime#1213L,dataChange#1214,stats#1233,tags#1216,deletionVector#1217,baseRowId#1218L,defaultRowCommitVersion#1219L,clusteringProvider#1220]