From 05c20f092ddeb63d89fb2466b8bee5f2b47a28cb Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Wed, 26 Aug 2026 12:26:22 +0100 Subject: [PATCH 01/24] HIVE-29743: Upgrade Calcite to 1.43.0 Initial test with 1.43.0-SNAPSHOT --- pom.xml | 10 +++- .../optimizer/calcite/HiveRelShuttleImpl.java | 54 +++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 038b51d3e14b..9977574d1bd0 100644 --- a/pom.xml +++ b/pom.xml @@ -119,7 +119,7 @@ 4.2.1 12.22.0 1.84 - 1.42.0 + 1.43.0-SNAPSHOT 6.0.5 6.0.11 3.2.0-release @@ -278,6 +278,14 @@ false + + + apache.snapshots + Apache Snapshots + https://repository.apache.org/content/repositories/snapshots/ + false + truedaily + diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelShuttleImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelShuttleImpl.java index 6f74664c0bd7..eb7176897dfa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelShuttleImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelShuttleImpl.java @@ -22,8 +22,17 @@ import java.util.Deque; import org.apache.calcite.linq4j.Ord; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Collect; +import org.apache.calcite.rel.core.Combine; +import org.apache.calcite.rel.core.ConditionalCorrelate; +import org.apache.calcite.rel.core.Sample; +import org.apache.calcite.rel.core.Snapshot; +import org.apache.calcite.rel.core.SortExchange; import org.apache.calcite.rel.core.TableFunctionScan; import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.core.TableSpool; +import org.apache.calcite.rel.core.Uncollect; +import org.apache.calcite.rel.core.Window; import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.logical.LogicalAsofJoin; import org.apache.calcite.rel.logical.LogicalCalc; @@ -190,6 +199,51 @@ public RelNode visit(LogicalAsofJoin logicalAsofJoin) { public RelNode visit(LogicalRepeatUnion logicalRepeatUnion) { return visitChildren(logicalRepeatUnion); } + + @Override + public RelNode visit(Window window) { + return visitChildren(window); + } + + @Override + public RelNode visit(Snapshot snapshot) { + return visitChildren(snapshot); + } + + @Override + public RelNode visit(Collect collect) { + return visitChildren(collect); + } + + @Override + public RelNode visit(Sample sample) { + return visitChildren(sample); + } + + @Override + public RelNode visit(Uncollect uncollect) { + return visitChildren(uncollect); + } + + @Override + public RelNode visit(Combine combine) { + return visitChildren(combine); + } + + @Override + public RelNode visit(ConditionalCorrelate conditionalCorrelate) { + return visitChildren(conditionalCorrelate); + } + + @Override + public RelNode visit(SortExchange sortExchange) { + return visitChildren(sortExchange); + } + + @Override + public RelNode visit(TableSpool tableSpool) { + return visitChildren(tableSpool); + } } // End RelShuttleImpl.java From d5401241054a306eac174c85d1ad11efe575e5c3 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Wed, 26 Aug 2026 13:11:34 +0100 Subject: [PATCH 02/24] Minor: fix format in pom file --- pom.xml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 9977574d1bd0..07c597f6b063 100644 --- a/pom.xml +++ b/pom.xml @@ -283,8 +283,13 @@ apache.snapshots Apache Snapshots https://repository.apache.org/content/repositories/snapshots/ - false - truedaily + + false + + + true + daily + From 2ffe6c87a86c9be501e17df674ef747a32b9bbf2 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Fri, 28 Aug 2026 11:26:33 +0100 Subject: [PATCH 03/24] Remove Apache repository configuration, not needed for CI (already taken care of at artifactory level) --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 07c597f6b063..870464b272fa 100644 --- a/pom.xml +++ b/pom.xml @@ -278,7 +278,7 @@ false - + From 7d89abcb246e4a6cd5407d462704d27dcd9c1972 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Tue, 1 Sep 2026 15:21:46 +0100 Subject: [PATCH 04/24] materialized_view_partitioned_2 fixed (as expected, due to fix CALCITE-7635) --- .../materialized_view_partitioned_2.q.out | 260 +----------------- 1 file changed, 7 insertions(+), 253 deletions(-) diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_2.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_2.q.out index 9d222fc5bd34..21bceb01fde5 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_2.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_2.q.out @@ -549,137 +549,13 @@ POSTHOOK: Output: default@partition_mv_4 PREHOOK: query: EXPLAIN SELECT * FROM src_txn_2 where key > 224 and key < 226 PREHOOK: type: QUERY -PREHOOK: Input: default@partition_mv_2 -PREHOOK: Input: default@partition_mv_2@value=val_201 -PREHOOK: Input: default@partition_mv_2@value=val_202 -PREHOOK: Input: default@partition_mv_2@value=val_203 -PREHOOK: Input: default@partition_mv_2@value=val_205 -PREHOOK: Input: default@partition_mv_2@value=val_207 -PREHOOK: Input: default@partition_mv_2@value=val_208 -PREHOOK: Input: default@partition_mv_2@value=val_209 -PREHOOK: Input: default@partition_mv_2@value=val_213 -PREHOOK: Input: default@partition_mv_2@value=val_214 -PREHOOK: Input: default@partition_mv_2@value=val_216 -PREHOOK: Input: default@partition_mv_2@value=val_217 -PREHOOK: Input: default@partition_mv_2@value=val_218 -PREHOOK: Input: default@partition_mv_2@value=val_219 -PREHOOK: Input: default@partition_mv_2@value=val_221 -PREHOOK: Input: default@partition_mv_2@value=val_222 -PREHOOK: Input: default@partition_mv_2@value=val_223 -PREHOOK: Input: default@partition_mv_2@value=val_224 -PREHOOK: Input: default@partition_mv_2@value=val_226 -PREHOOK: Input: default@partition_mv_2@value=val_228 -PREHOOK: Input: default@partition_mv_2@value=val_229 -PREHOOK: Input: default@partition_mv_2@value=val_230 -PREHOOK: Input: default@partition_mv_2@value=val_233 -PREHOOK: Input: default@partition_mv_2@value=val_235 -PREHOOK: Input: default@partition_mv_2@value=val_237 -PREHOOK: Input: default@partition_mv_2@value=val_238 -PREHOOK: Input: default@partition_mv_2@value=val_239 -PREHOOK: Input: default@partition_mv_2@value=val_241 -PREHOOK: Input: default@partition_mv_2@value=val_242 -PREHOOK: Input: default@partition_mv_2@value=val_244 -PREHOOK: Input: default@partition_mv_2@value=val_247 -PREHOOK: Input: default@partition_mv_2@value=val_248 -PREHOOK: Input: default@partition_mv_2@value=val_249 -PREHOOK: Input: default@partition_mv_2@value=val_252 -PREHOOK: Input: default@partition_mv_2@value=val_255 -PREHOOK: Input: default@partition_mv_2@value=val_256 -PREHOOK: Input: default@partition_mv_2@value=val_257 -PREHOOK: Input: default@partition_mv_2@value=val_258 -PREHOOK: Input: default@partition_mv_2@value=val_260 -PREHOOK: Input: default@partition_mv_2@value=val_262 -PREHOOK: Input: default@partition_mv_2@value=val_263 -PREHOOK: Input: default@partition_mv_2@value=val_265 -PREHOOK: Input: default@partition_mv_2@value=val_266 -PREHOOK: Input: default@partition_mv_2@value=val_272 -PREHOOK: Input: default@partition_mv_2@value=val_273 -PREHOOK: Input: default@partition_mv_2@value=val_274 -PREHOOK: Input: default@partition_mv_2@value=val_275 -PREHOOK: Input: default@partition_mv_2@value=val_277 -PREHOOK: Input: default@partition_mv_2@value=val_278 -PREHOOK: Input: default@partition_mv_2@value=val_280 -PREHOOK: Input: default@partition_mv_2@value=val_281 -PREHOOK: Input: default@partition_mv_2@value=val_282 -PREHOOK: Input: default@partition_mv_2@value=val_283 -PREHOOK: Input: default@partition_mv_2@value=val_284 -PREHOOK: Input: default@partition_mv_2@value=val_285 -PREHOOK: Input: default@partition_mv_2@value=val_286 -PREHOOK: Input: default@partition_mv_2@value=val_287 -PREHOOK: Input: default@partition_mv_2@value=val_288 -PREHOOK: Input: default@partition_mv_2@value=val_289 -PREHOOK: Input: default@partition_mv_2@value=val_291 -PREHOOK: Input: default@partition_mv_2@value=val_292 -PREHOOK: Input: default@partition_mv_2@value=val_296 -PREHOOK: Input: default@partition_mv_2@value=val_298 +PREHOOK: Input: default@partition_mv_3 PREHOOK: Input: default@src_txn_2 #### A masked pattern was here #### POSTHOOK: query: EXPLAIN SELECT * FROM src_txn_2 where key > 224 and key < 226 POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_mv_2 -POSTHOOK: Input: default@partition_mv_2@value=val_201 -POSTHOOK: Input: default@partition_mv_2@value=val_202 -POSTHOOK: Input: default@partition_mv_2@value=val_203 -POSTHOOK: Input: default@partition_mv_2@value=val_205 -POSTHOOK: Input: default@partition_mv_2@value=val_207 -POSTHOOK: Input: default@partition_mv_2@value=val_208 -POSTHOOK: Input: default@partition_mv_2@value=val_209 -POSTHOOK: Input: default@partition_mv_2@value=val_213 -POSTHOOK: Input: default@partition_mv_2@value=val_214 -POSTHOOK: Input: default@partition_mv_2@value=val_216 -POSTHOOK: Input: default@partition_mv_2@value=val_217 -POSTHOOK: Input: default@partition_mv_2@value=val_218 -POSTHOOK: Input: default@partition_mv_2@value=val_219 -POSTHOOK: Input: default@partition_mv_2@value=val_221 -POSTHOOK: Input: default@partition_mv_2@value=val_222 -POSTHOOK: Input: default@partition_mv_2@value=val_223 -POSTHOOK: Input: default@partition_mv_2@value=val_224 -POSTHOOK: Input: default@partition_mv_2@value=val_226 -POSTHOOK: Input: default@partition_mv_2@value=val_228 -POSTHOOK: Input: default@partition_mv_2@value=val_229 -POSTHOOK: Input: default@partition_mv_2@value=val_230 -POSTHOOK: Input: default@partition_mv_2@value=val_233 -POSTHOOK: Input: default@partition_mv_2@value=val_235 -POSTHOOK: Input: default@partition_mv_2@value=val_237 -POSTHOOK: Input: default@partition_mv_2@value=val_238 -POSTHOOK: Input: default@partition_mv_2@value=val_239 -POSTHOOK: Input: default@partition_mv_2@value=val_241 -POSTHOOK: Input: default@partition_mv_2@value=val_242 -POSTHOOK: Input: default@partition_mv_2@value=val_244 -POSTHOOK: Input: default@partition_mv_2@value=val_247 -POSTHOOK: Input: default@partition_mv_2@value=val_248 -POSTHOOK: Input: default@partition_mv_2@value=val_249 -POSTHOOK: Input: default@partition_mv_2@value=val_252 -POSTHOOK: Input: default@partition_mv_2@value=val_255 -POSTHOOK: Input: default@partition_mv_2@value=val_256 -POSTHOOK: Input: default@partition_mv_2@value=val_257 -POSTHOOK: Input: default@partition_mv_2@value=val_258 -POSTHOOK: Input: default@partition_mv_2@value=val_260 -POSTHOOK: Input: default@partition_mv_2@value=val_262 -POSTHOOK: Input: default@partition_mv_2@value=val_263 -POSTHOOK: Input: default@partition_mv_2@value=val_265 -POSTHOOK: Input: default@partition_mv_2@value=val_266 -POSTHOOK: Input: default@partition_mv_2@value=val_272 -POSTHOOK: Input: default@partition_mv_2@value=val_273 -POSTHOOK: Input: default@partition_mv_2@value=val_274 -POSTHOOK: Input: default@partition_mv_2@value=val_275 -POSTHOOK: Input: default@partition_mv_2@value=val_277 -POSTHOOK: Input: default@partition_mv_2@value=val_278 -POSTHOOK: Input: default@partition_mv_2@value=val_280 -POSTHOOK: Input: default@partition_mv_2@value=val_281 -POSTHOOK: Input: default@partition_mv_2@value=val_282 -POSTHOOK: Input: default@partition_mv_2@value=val_283 -POSTHOOK: Input: default@partition_mv_2@value=val_284 -POSTHOOK: Input: default@partition_mv_2@value=val_285 -POSTHOOK: Input: default@partition_mv_2@value=val_286 -POSTHOOK: Input: default@partition_mv_2@value=val_287 -POSTHOOK: Input: default@partition_mv_2@value=val_288 -POSTHOOK: Input: default@partition_mv_2@value=val_289 -POSTHOOK: Input: default@partition_mv_2@value=val_291 -POSTHOOK: Input: default@partition_mv_2@value=val_292 -POSTHOOK: Input: default@partition_mv_2@value=val_296 -POSTHOOK: Input: default@partition_mv_2@value=val_298 +POSTHOOK: Input: default@partition_mv_3 POSTHOOK: Input: default@src_txn_2 #### A masked pattern was here #### STAGE DEPENDENCIES: @@ -691,7 +567,7 @@ STAGE PLANS: limit: -1 Processor Tree: TableScan - alias: default.partition_mv_2 + alias: default.partition_mv_3 filterExpr: ((UDFToDouble(key) > 224.0D) and (UDFToDouble(key) < 226.0D)) (type: boolean) Filter Operator predicate: ((UDFToDouble(key) > 224.0D) and (UDFToDouble(key) < 226.0D)) (type: boolean) @@ -702,136 +578,14 @@ STAGE PLANS: PREHOOK: query: SELECT * FROM src_txn_2 where key > 223 and key < 225 PREHOOK: type: QUERY -PREHOOK: Input: default@partition_mv_2 -PREHOOK: Input: default@partition_mv_2@value=val_201 -PREHOOK: Input: default@partition_mv_2@value=val_202 -PREHOOK: Input: default@partition_mv_2@value=val_203 -PREHOOK: Input: default@partition_mv_2@value=val_205 -PREHOOK: Input: default@partition_mv_2@value=val_207 -PREHOOK: Input: default@partition_mv_2@value=val_208 -PREHOOK: Input: default@partition_mv_2@value=val_209 -PREHOOK: Input: default@partition_mv_2@value=val_213 -PREHOOK: Input: default@partition_mv_2@value=val_214 -PREHOOK: Input: default@partition_mv_2@value=val_216 -PREHOOK: Input: default@partition_mv_2@value=val_217 -PREHOOK: Input: default@partition_mv_2@value=val_218 -PREHOOK: Input: default@partition_mv_2@value=val_219 -PREHOOK: Input: default@partition_mv_2@value=val_221 -PREHOOK: Input: default@partition_mv_2@value=val_222 -PREHOOK: Input: default@partition_mv_2@value=val_223 -PREHOOK: Input: default@partition_mv_2@value=val_224 -PREHOOK: Input: default@partition_mv_2@value=val_226 -PREHOOK: Input: default@partition_mv_2@value=val_228 -PREHOOK: Input: default@partition_mv_2@value=val_229 -PREHOOK: Input: default@partition_mv_2@value=val_230 -PREHOOK: Input: default@partition_mv_2@value=val_233 -PREHOOK: Input: default@partition_mv_2@value=val_235 -PREHOOK: Input: default@partition_mv_2@value=val_237 -PREHOOK: Input: default@partition_mv_2@value=val_238 -PREHOOK: Input: default@partition_mv_2@value=val_239 -PREHOOK: Input: default@partition_mv_2@value=val_241 -PREHOOK: Input: default@partition_mv_2@value=val_242 -PREHOOK: Input: default@partition_mv_2@value=val_244 -PREHOOK: Input: default@partition_mv_2@value=val_247 -PREHOOK: Input: default@partition_mv_2@value=val_248 -PREHOOK: Input: default@partition_mv_2@value=val_249 -PREHOOK: Input: default@partition_mv_2@value=val_252 -PREHOOK: Input: default@partition_mv_2@value=val_255 -PREHOOK: Input: default@partition_mv_2@value=val_256 -PREHOOK: Input: default@partition_mv_2@value=val_257 -PREHOOK: Input: default@partition_mv_2@value=val_258 -PREHOOK: Input: default@partition_mv_2@value=val_260 -PREHOOK: Input: default@partition_mv_2@value=val_262 -PREHOOK: Input: default@partition_mv_2@value=val_263 -PREHOOK: Input: default@partition_mv_2@value=val_265 -PREHOOK: Input: default@partition_mv_2@value=val_266 -PREHOOK: Input: default@partition_mv_2@value=val_272 -PREHOOK: Input: default@partition_mv_2@value=val_273 -PREHOOK: Input: default@partition_mv_2@value=val_274 -PREHOOK: Input: default@partition_mv_2@value=val_275 -PREHOOK: Input: default@partition_mv_2@value=val_277 -PREHOOK: Input: default@partition_mv_2@value=val_278 -PREHOOK: Input: default@partition_mv_2@value=val_280 -PREHOOK: Input: default@partition_mv_2@value=val_281 -PREHOOK: Input: default@partition_mv_2@value=val_282 -PREHOOK: Input: default@partition_mv_2@value=val_283 -PREHOOK: Input: default@partition_mv_2@value=val_284 -PREHOOK: Input: default@partition_mv_2@value=val_285 -PREHOOK: Input: default@partition_mv_2@value=val_286 -PREHOOK: Input: default@partition_mv_2@value=val_287 -PREHOOK: Input: default@partition_mv_2@value=val_288 -PREHOOK: Input: default@partition_mv_2@value=val_289 -PREHOOK: Input: default@partition_mv_2@value=val_291 -PREHOOK: Input: default@partition_mv_2@value=val_292 -PREHOOK: Input: default@partition_mv_2@value=val_296 -PREHOOK: Input: default@partition_mv_2@value=val_298 +PREHOOK: Input: default@partition_mv_3 +PREHOOK: Input: default@partition_mv_3@key=224 PREHOOK: Input: default@src_txn_2 #### A masked pattern was here #### POSTHOOK: query: SELECT * FROM src_txn_2 where key > 223 and key < 225 POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_mv_2 -POSTHOOK: Input: default@partition_mv_2@value=val_201 -POSTHOOK: Input: default@partition_mv_2@value=val_202 -POSTHOOK: Input: default@partition_mv_2@value=val_203 -POSTHOOK: Input: default@partition_mv_2@value=val_205 -POSTHOOK: Input: default@partition_mv_2@value=val_207 -POSTHOOK: Input: default@partition_mv_2@value=val_208 -POSTHOOK: Input: default@partition_mv_2@value=val_209 -POSTHOOK: Input: default@partition_mv_2@value=val_213 -POSTHOOK: Input: default@partition_mv_2@value=val_214 -POSTHOOK: Input: default@partition_mv_2@value=val_216 -POSTHOOK: Input: default@partition_mv_2@value=val_217 -POSTHOOK: Input: default@partition_mv_2@value=val_218 -POSTHOOK: Input: default@partition_mv_2@value=val_219 -POSTHOOK: Input: default@partition_mv_2@value=val_221 -POSTHOOK: Input: default@partition_mv_2@value=val_222 -POSTHOOK: Input: default@partition_mv_2@value=val_223 -POSTHOOK: Input: default@partition_mv_2@value=val_224 -POSTHOOK: Input: default@partition_mv_2@value=val_226 -POSTHOOK: Input: default@partition_mv_2@value=val_228 -POSTHOOK: Input: default@partition_mv_2@value=val_229 -POSTHOOK: Input: default@partition_mv_2@value=val_230 -POSTHOOK: Input: default@partition_mv_2@value=val_233 -POSTHOOK: Input: default@partition_mv_2@value=val_235 -POSTHOOK: Input: default@partition_mv_2@value=val_237 -POSTHOOK: Input: default@partition_mv_2@value=val_238 -POSTHOOK: Input: default@partition_mv_2@value=val_239 -POSTHOOK: Input: default@partition_mv_2@value=val_241 -POSTHOOK: Input: default@partition_mv_2@value=val_242 -POSTHOOK: Input: default@partition_mv_2@value=val_244 -POSTHOOK: Input: default@partition_mv_2@value=val_247 -POSTHOOK: Input: default@partition_mv_2@value=val_248 -POSTHOOK: Input: default@partition_mv_2@value=val_249 -POSTHOOK: Input: default@partition_mv_2@value=val_252 -POSTHOOK: Input: default@partition_mv_2@value=val_255 -POSTHOOK: Input: default@partition_mv_2@value=val_256 -POSTHOOK: Input: default@partition_mv_2@value=val_257 -POSTHOOK: Input: default@partition_mv_2@value=val_258 -POSTHOOK: Input: default@partition_mv_2@value=val_260 -POSTHOOK: Input: default@partition_mv_2@value=val_262 -POSTHOOK: Input: default@partition_mv_2@value=val_263 -POSTHOOK: Input: default@partition_mv_2@value=val_265 -POSTHOOK: Input: default@partition_mv_2@value=val_266 -POSTHOOK: Input: default@partition_mv_2@value=val_272 -POSTHOOK: Input: default@partition_mv_2@value=val_273 -POSTHOOK: Input: default@partition_mv_2@value=val_274 -POSTHOOK: Input: default@partition_mv_2@value=val_275 -POSTHOOK: Input: default@partition_mv_2@value=val_277 -POSTHOOK: Input: default@partition_mv_2@value=val_278 -POSTHOOK: Input: default@partition_mv_2@value=val_280 -POSTHOOK: Input: default@partition_mv_2@value=val_281 -POSTHOOK: Input: default@partition_mv_2@value=val_282 -POSTHOOK: Input: default@partition_mv_2@value=val_283 -POSTHOOK: Input: default@partition_mv_2@value=val_284 -POSTHOOK: Input: default@partition_mv_2@value=val_285 -POSTHOOK: Input: default@partition_mv_2@value=val_286 -POSTHOOK: Input: default@partition_mv_2@value=val_287 -POSTHOOK: Input: default@partition_mv_2@value=val_288 -POSTHOOK: Input: default@partition_mv_2@value=val_289 -POSTHOOK: Input: default@partition_mv_2@value=val_291 -POSTHOOK: Input: default@partition_mv_2@value=val_292 -POSTHOOK: Input: default@partition_mv_2@value=val_296 -POSTHOOK: Input: default@partition_mv_2@value=val_298 +POSTHOOK: Input: default@partition_mv_3 +POSTHOOK: Input: default@partition_mv_3@key=224 POSTHOOK: Input: default@src_txn_2 #### A masked pattern was here #### 224 val_224 From af03d2adee89f5e4f4f3f000c114d1290a230022 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Tue, 1 Sep 2026 17:20:20 +0100 Subject: [PATCH 05/24] Adjust test plan vector_interval_2.q.out (same filter, different terms order) --- .../llap/vector_interval_2.q.out | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ql/src/test/results/clientpositive/llap/vector_interval_2.q.out b/ql/src/test/results/clientpositive/llap/vector_interval_2.q.out index 91f4610f0508..7529209b4812 100644 --- a/ql/src/test/results/clientpositive/llap/vector_interval_2.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_interval_2.q.out @@ -962,7 +962,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: vector_interval_2 - filterExpr: ((CAST( str2 AS INTERVAL YEAR TO MONTH) > INTERVAL'1-2') and (CAST( str1 AS INTERVAL YEAR TO MONTH) <= CAST( str2 AS INTERVAL YEAR TO MONTH)) and (CAST( str1 AS INTERVAL YEAR TO MONTH) < CAST( str2 AS INTERVAL YEAR TO MONTH)) and (CAST( str2 AS INTERVAL YEAR TO MONTH) >= CAST( str1 AS INTERVAL YEAR TO MONTH)) and (CAST( str2 AS INTERVAL YEAR TO MONTH) > CAST( str1 AS INTERVAL YEAR TO MONTH)) and (CAST( str1 AS INTERVAL YEAR TO MONTH) <> CAST( str2 AS INTERVAL YEAR TO MONTH)) and (CAST( str1 AS INTERVAL YEAR TO MONTH) = INTERVAL'1-2')) (type: boolean) + filterExpr: ((CAST( str2 AS INTERVAL YEAR TO MONTH) > INTERVAL'1-2') and (CAST( str1 AS INTERVAL YEAR TO MONTH) <= CAST( str2 AS INTERVAL YEAR TO MONTH)) and (CAST( str1 AS INTERVAL YEAR TO MONTH) < CAST( str2 AS INTERVAL YEAR TO MONTH)) and (CAST( str2 AS INTERVAL YEAR TO MONTH) >= CAST( str1 AS INTERVAL YEAR TO MONTH)) and (CAST( str2 AS INTERVAL YEAR TO MONTH) > CAST( str1 AS INTERVAL YEAR TO MONTH)) and (CAST( str1 AS INTERVAL YEAR TO MONTH) = INTERVAL'1-2') and (CAST( str1 AS INTERVAL YEAR TO MONTH) <> CAST( str2 AS INTERVAL YEAR TO MONTH))) (type: boolean) Statistics: Num rows: 2 Data size: 428 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -970,8 +970,8 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: FilterIntervalYearMonthColGreaterIntervalYearMonthScalar(col 8:interval_year_month, val 14)(children: CastStringToIntervalYearMonth(col 3:string) -> 8:interval_year_month), FilterLongColLessEqualLongColumn(col 9:interval_year_month, col 10:interval_year_month)(children: CastStringToIntervalYearMonth(col 2:string) -> 9:interval_year_month, CastStringToIntervalYearMonth(col 3:string) -> 10:interval_year_month), FilterLongColLessLongColumn(col 11:interval_year_month, col 12:interval_year_month)(children: CastStringToIntervalYearMonth(col 2:string) -> 11:interval_year_month, CastStringToIntervalYearMonth(col 3:string) -> 12:interval_year_month), FilterLongColGreaterEqualLongColumn(col 13:interval_year_month, col 14:interval_year_month)(children: CastStringToIntervalYearMonth(col 3:string) -> 13:interval_year_month, CastStringToIntervalYearMonth(col 2:string) -> 14:interval_year_month), FilterLongColGreaterLongColumn(col 15:interval_year_month, col 16:interval_year_month)(children: CastStringToIntervalYearMonth(col 3:string) -> 15:interval_year_month, CastStringToIntervalYearMonth(col 2:string) -> 16:interval_year_month), FilterLongColNotEqualLongColumn(col 17:interval_year_month, col 18:interval_year_month)(children: CastStringToIntervalYearMonth(col 2:string) -> 17:interval_year_month, CastStringToIntervalYearMonth(col 3:string) -> 18:interval_year_month), FilterIntervalYearMonthColEqualIntervalYearMonthScalar(col 19:interval_year_month, val 14)(children: CastStringToIntervalYearMonth(col 2:string) -> 19:interval_year_month)) - predicate: ((CAST( str2 AS INTERVAL YEAR TO MONTH) > INTERVAL'1-2') and (CAST( str1 AS INTERVAL YEAR TO MONTH) <= CAST( str2 AS INTERVAL YEAR TO MONTH)) and (CAST( str1 AS INTERVAL YEAR TO MONTH) < CAST( str2 AS INTERVAL YEAR TO MONTH)) and (CAST( str2 AS INTERVAL YEAR TO MONTH) >= CAST( str1 AS INTERVAL YEAR TO MONTH)) and (CAST( str2 AS INTERVAL YEAR TO MONTH) > CAST( str1 AS INTERVAL YEAR TO MONTH)) and (CAST( str1 AS INTERVAL YEAR TO MONTH) <> CAST( str2 AS INTERVAL YEAR TO MONTH)) and (CAST( str1 AS INTERVAL YEAR TO MONTH) = INTERVAL'1-2')) (type: boolean) + predicateExpression: FilterExprAndExpr(children: FilterIntervalYearMonthColGreaterIntervalYearMonthScalar(col 8:interval_year_month, val 14)(children: CastStringToIntervalYearMonth(col 3:string) -> 8:interval_year_month), FilterLongColLessEqualLongColumn(col 9:interval_year_month, col 10:interval_year_month)(children: CastStringToIntervalYearMonth(col 2:string) -> 9:interval_year_month, CastStringToIntervalYearMonth(col 3:string) -> 10:interval_year_month), FilterLongColLessLongColumn(col 11:interval_year_month, col 12:interval_year_month)(children: CastStringToIntervalYearMonth(col 2:string) -> 11:interval_year_month, CastStringToIntervalYearMonth(col 3:string) -> 12:interval_year_month), FilterLongColGreaterEqualLongColumn(col 13:interval_year_month, col 14:interval_year_month)(children: CastStringToIntervalYearMonth(col 3:string) -> 13:interval_year_month, CastStringToIntervalYearMonth(col 2:string) -> 14:interval_year_month), FilterLongColGreaterLongColumn(col 15:interval_year_month, col 16:interval_year_month)(children: CastStringToIntervalYearMonth(col 3:string) -> 15:interval_year_month, CastStringToIntervalYearMonth(col 2:string) -> 16:interval_year_month), FilterIntervalYearMonthColEqualIntervalYearMonthScalar(col 17:interval_year_month, val 14)(children: CastStringToIntervalYearMonth(col 2:string) -> 17:interval_year_month), FilterLongColNotEqualLongColumn(col 18:interval_year_month, col 19:interval_year_month)(children: CastStringToIntervalYearMonth(col 2:string) -> 18:interval_year_month, CastStringToIntervalYearMonth(col 3:string) -> 19:interval_year_month)) + predicate: ((CAST( str2 AS INTERVAL YEAR TO MONTH) > INTERVAL'1-2') and (CAST( str1 AS INTERVAL YEAR TO MONTH) <= CAST( str2 AS INTERVAL YEAR TO MONTH)) and (CAST( str1 AS INTERVAL YEAR TO MONTH) < CAST( str2 AS INTERVAL YEAR TO MONTH)) and (CAST( str2 AS INTERVAL YEAR TO MONTH) >= CAST( str1 AS INTERVAL YEAR TO MONTH)) and (CAST( str2 AS INTERVAL YEAR TO MONTH) > CAST( str1 AS INTERVAL YEAR TO MONTH)) and (CAST( str1 AS INTERVAL YEAR TO MONTH) = INTERVAL'1-2') and (CAST( str1 AS INTERVAL YEAR TO MONTH) <> CAST( str2 AS INTERVAL YEAR TO MONTH))) (type: boolean) Statistics: Num rows: 1 Data size: 214 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ts (type: timestamp) @@ -1162,7 +1162,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: vector_interval_2 - filterExpr: ((CAST( str4 AS INTERVAL DAY TO SECOND) > INTERVAL'1 02:03:04.000000000') and (CAST( str3 AS INTERVAL DAY TO SECOND) <= CAST( str4 AS INTERVAL DAY TO SECOND)) and (CAST( str3 AS INTERVAL DAY TO SECOND) < CAST( str4 AS INTERVAL DAY TO SECOND)) and (CAST( str4 AS INTERVAL DAY TO SECOND) >= CAST( str3 AS INTERVAL DAY TO SECOND)) and (CAST( str4 AS INTERVAL DAY TO SECOND) > CAST( str3 AS INTERVAL DAY TO SECOND)) and (CAST( str3 AS INTERVAL DAY TO SECOND) <> CAST( str4 AS INTERVAL DAY TO SECOND)) and (CAST( str3 AS INTERVAL DAY TO SECOND) = INTERVAL'1 02:03:04.000000000')) (type: boolean) + filterExpr: ((CAST( str4 AS INTERVAL DAY TO SECOND) > INTERVAL'1 02:03:04.000000000') and (CAST( str3 AS INTERVAL DAY TO SECOND) <= CAST( str4 AS INTERVAL DAY TO SECOND)) and (CAST( str3 AS INTERVAL DAY TO SECOND) < CAST( str4 AS INTERVAL DAY TO SECOND)) and (CAST( str4 AS INTERVAL DAY TO SECOND) >= CAST( str3 AS INTERVAL DAY TO SECOND)) and (CAST( str4 AS INTERVAL DAY TO SECOND) > CAST( str3 AS INTERVAL DAY TO SECOND)) and (CAST( str3 AS INTERVAL DAY TO SECOND) = INTERVAL'1 02:03:04.000000000') and (CAST( str3 AS INTERVAL DAY TO SECOND) <> CAST( str4 AS INTERVAL DAY TO SECOND))) (type: boolean) Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -1170,8 +1170,8 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: FilterIntervalDayTimeColGreaterIntervalDayTimeScalar(col 8:interval_day_time, val 1 02:03:04.000000000)(children: CastStringToIntervalDayTime(col 5:string) -> 8:interval_day_time), FilterIntervalDayTimeColLessEqualIntervalDayTimeColumn(col 9:interval_day_time, col 10:interval_day_time)(children: CastStringToIntervalDayTime(col 4:string) -> 9:interval_day_time, CastStringToIntervalDayTime(col 5:string) -> 10:interval_day_time), FilterIntervalDayTimeColLessIntervalDayTimeColumn(col 11:interval_day_time, col 12:interval_day_time)(children: CastStringToIntervalDayTime(col 4:string) -> 11:interval_day_time, CastStringToIntervalDayTime(col 5:string) -> 12:interval_day_time), FilterIntervalDayTimeColGreaterEqualIntervalDayTimeColumn(col 13:interval_day_time, col 14:interval_day_time)(children: CastStringToIntervalDayTime(col 5:string) -> 13:interval_day_time, CastStringToIntervalDayTime(col 4:string) -> 14:interval_day_time), FilterIntervalDayTimeColGreaterIntervalDayTimeColumn(col 15:interval_day_time, col 16:interval_day_time)(children: CastStringToIntervalDayTime(col 5:string) -> 15:interval_day_time, CastStringToIntervalDayTime(col 4:string) -> 16:interval_day_time), FilterIntervalDayTimeColNotEqualIntervalDayTimeColumn(col 17:interval_day_time, col 18:interval_day_time)(children: CastStringToIntervalDayTime(col 4:string) -> 17:interval_day_time, CastStringToIntervalDayTime(col 5:string) -> 18:interval_day_time), FilterIntervalDayTimeColEqualIntervalDayTimeScalar(col 19:interval_day_time, val 1 02:03:04.000000000)(children: CastStringToIntervalDayTime(col 4:string) -> 19:interval_day_time)) - predicate: ((CAST( str4 AS INTERVAL DAY TO SECOND) > INTERVAL'1 02:03:04.000000000') and (CAST( str3 AS INTERVAL DAY TO SECOND) <= CAST( str4 AS INTERVAL DAY TO SECOND)) and (CAST( str3 AS INTERVAL DAY TO SECOND) < CAST( str4 AS INTERVAL DAY TO SECOND)) and (CAST( str4 AS INTERVAL DAY TO SECOND) >= CAST( str3 AS INTERVAL DAY TO SECOND)) and (CAST( str4 AS INTERVAL DAY TO SECOND) > CAST( str3 AS INTERVAL DAY TO SECOND)) and (CAST( str3 AS INTERVAL DAY TO SECOND) <> CAST( str4 AS INTERVAL DAY TO SECOND)) and (CAST( str3 AS INTERVAL DAY TO SECOND) = INTERVAL'1 02:03:04.000000000')) (type: boolean) + predicateExpression: FilterExprAndExpr(children: FilterIntervalDayTimeColGreaterIntervalDayTimeScalar(col 8:interval_day_time, val 1 02:03:04.000000000)(children: CastStringToIntervalDayTime(col 5:string) -> 8:interval_day_time), FilterIntervalDayTimeColLessEqualIntervalDayTimeColumn(col 9:interval_day_time, col 10:interval_day_time)(children: CastStringToIntervalDayTime(col 4:string) -> 9:interval_day_time, CastStringToIntervalDayTime(col 5:string) -> 10:interval_day_time), FilterIntervalDayTimeColLessIntervalDayTimeColumn(col 11:interval_day_time, col 12:interval_day_time)(children: CastStringToIntervalDayTime(col 4:string) -> 11:interval_day_time, CastStringToIntervalDayTime(col 5:string) -> 12:interval_day_time), FilterIntervalDayTimeColGreaterEqualIntervalDayTimeColumn(col 13:interval_day_time, col 14:interval_day_time)(children: CastStringToIntervalDayTime(col 5:string) -> 13:interval_day_time, CastStringToIntervalDayTime(col 4:string) -> 14:interval_day_time), FilterIntervalDayTimeColGreaterIntervalDayTimeColumn(col 15:interval_day_time, col 16:interval_day_time)(children: CastStringToIntervalDayTime(col 5:string) -> 15:interval_day_time, CastStringToIntervalDayTime(col 4:string) -> 16:interval_day_time), FilterIntervalDayTimeColEqualIntervalDayTimeScalar(col 17:interval_day_time, val 1 02:03:04.000000000)(children: CastStringToIntervalDayTime(col 4:string) -> 17:interval_day_time), FilterIntervalDayTimeColNotEqualIntervalDayTimeColumn(col 18:interval_day_time, col 19:interval_day_time)(children: CastStringToIntervalDayTime(col 4:string) -> 18:interval_day_time, CastStringToIntervalDayTime(col 5:string) -> 19:interval_day_time)) + predicate: ((CAST( str4 AS INTERVAL DAY TO SECOND) > INTERVAL'1 02:03:04.000000000') and (CAST( str3 AS INTERVAL DAY TO SECOND) <= CAST( str4 AS INTERVAL DAY TO SECOND)) and (CAST( str3 AS INTERVAL DAY TO SECOND) < CAST( str4 AS INTERVAL DAY TO SECOND)) and (CAST( str4 AS INTERVAL DAY TO SECOND) >= CAST( str3 AS INTERVAL DAY TO SECOND)) and (CAST( str4 AS INTERVAL DAY TO SECOND) > CAST( str3 AS INTERVAL DAY TO SECOND)) and (CAST( str3 AS INTERVAL DAY TO SECOND) = INTERVAL'1 02:03:04.000000000') and (CAST( str3 AS INTERVAL DAY TO SECOND) <> CAST( str4 AS INTERVAL DAY TO SECOND))) (type: boolean) Statistics: Num rows: 1 Data size: 222 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ts (type: timestamp) @@ -1352,7 +1352,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: vector_interval_2 - filterExpr: (((dt + CAST( str1 AS INTERVAL YEAR TO MONTH)) = DATE'2002-03-01') and (dt <> (dt + CAST( str1 AS INTERVAL YEAR TO MONTH))) and ((dt + INTERVAL'1-2') = DATE'2002-03-01') and (dt <> (dt + INTERVAL'1-2'))) (type: boolean) + filterExpr: ((DATE'2002-03-01' = (dt + CAST( str1 AS INTERVAL YEAR TO MONTH))) and (dt <> (dt + CAST( str1 AS INTERVAL YEAR TO MONTH))) and (DATE'2002-03-01' = (dt + INTERVAL'1-2')) and (dt <> (dt + INTERVAL'1-2'))) (type: boolean) Statistics: Num rows: 2 Data size: 366 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -1360,8 +1360,8 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: FilterDateColEqualDateScalar(col 9:date, val 11747)(children: DateColAddIntervalYearMonthColumn(col 1:date, col 8:interval_year_month)(children: CastStringToIntervalYearMonth(col 2:string) -> 8:interval_year_month) -> 9:date), FilterLongColNotEqualLongColumn(col 1:date, col 11:date)(children: DateColAddIntervalYearMonthColumn(col 1:date, col 10:interval_year_month)(children: CastStringToIntervalYearMonth(col 2:string) -> 10:interval_year_month) -> 11:date), FilterDateColEqualDateScalar(col 12:date, val 11747)(children: DateColAddIntervalYearMonthScalar(col 1:date, val 1-2) -> 12:date), FilterLongColNotEqualLongColumn(col 1:date, col 13:date)(children: DateColAddIntervalYearMonthScalar(col 1:date, val 1-2) -> 13:date)) - predicate: (((dt + CAST( str1 AS INTERVAL YEAR TO MONTH)) = DATE'2002-03-01') and (dt <> (dt + CAST( str1 AS INTERVAL YEAR TO MONTH))) and ((dt + INTERVAL'1-2') = DATE'2002-03-01') and (dt <> (dt + INTERVAL'1-2'))) (type: boolean) + predicateExpression: FilterExprAndExpr(children: FilterDateScalarEqualDateColumn(val 11747, col 9:date)(children: DateColAddIntervalYearMonthColumn(col 1:date, col 8:interval_year_month)(children: CastStringToIntervalYearMonth(col 2:string) -> 8:interval_year_month) -> 9:date), FilterLongColNotEqualLongColumn(col 1:date, col 11:date)(children: DateColAddIntervalYearMonthColumn(col 1:date, col 10:interval_year_month)(children: CastStringToIntervalYearMonth(col 2:string) -> 10:interval_year_month) -> 11:date), FilterDateScalarEqualDateColumn(val 11747, col 12:date)(children: DateColAddIntervalYearMonthScalar(col 1:date, val 1-2) -> 12:date), FilterLongColNotEqualLongColumn(col 1:date, col 13:date)(children: DateColAddIntervalYearMonthScalar(col 1:date, val 1-2) -> 13:date)) + predicate: ((DATE'2002-03-01' = (dt + CAST( str1 AS INTERVAL YEAR TO MONTH))) and (dt <> (dt + CAST( str1 AS INTERVAL YEAR TO MONTH))) and (DATE'2002-03-01' = (dt + INTERVAL'1-2')) and (dt <> (dt + INTERVAL'1-2'))) (type: boolean) Statistics: Num rows: 1 Data size: 183 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ts (type: timestamp) From b7481345072bdee59202226f1d238693774de848 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Tue, 1 Sep 2026 17:21:10 +0100 Subject: [PATCH 06/24] Adjust test plans (simplifications from CALCITE-7722) --- .../llap/allcolref_in_udf.q.out | 4 +- .../clientpositive/llap/auto_join13.q.out | 3 +- .../clientpositive/llap/interval_3.q.out | 4 +- .../results/clientpositive/llap/join13.q.out | 1 - .../clientpositive/llap/subquery_notin.q.out | 2 +- .../llap/vector_interval_mapjoin.q.out | 38 +++++++++---------- 6 files changed, 25 insertions(+), 27 deletions(-) diff --git a/ql/src/test/results/clientpositive/llap/allcolref_in_udf.q.out b/ql/src/test/results/clientpositive/llap/allcolref_in_udf.q.out index 9b6626ad5dc7..236d52a2ded5 100644 --- a/ql/src/test/results/clientpositive/llap/allcolref_in_udf.q.out +++ b/ql/src/test/results/clientpositive/llap/allcolref_in_udf.q.out @@ -76,10 +76,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - filterExpr: ((UDFToDouble(key) < 100.0D) and (UDFToDouble(key) + 1.0D) is not null) (type: boolean) + filterExpr: (UDFToDouble(key) < 100.0D) (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((UDFToDouble(key) < 100.0D) and (UDFToDouble(key) + 1.0D) is not null) (type: boolean) + predicate: (UDFToDouble(key) < 100.0D) (type: boolean) Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string), concat(key, value) (type: string), (UDFToDouble(key) + 1.0D) (type: double) diff --git a/ql/src/test/results/clientpositive/llap/auto_join13.q.out b/ql/src/test/results/clientpositive/llap/auto_join13.q.out index 55352923f3e6..8d7b32361f23 100644 --- a/ql/src/test/results/clientpositive/llap/auto_join13.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_join13.q.out @@ -61,7 +61,6 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 input vertices: 1 Reducer 4 - residual filter predicates: {(UDFToDouble(_col2) + UDFToDouble(_col0)) is not null} Statistics: Num rows: 166 Data size: 43990 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -88,7 +87,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Execution mode: llap + Execution mode: vectorized, llap LLAP IO: all inputs Map 3 Map Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/interval_3.q.out b/ql/src/test/results/clientpositive/llap/interval_3.q.out index 40a8a3df742b..f004b3e990a2 100644 --- a/ql/src/test/results/clientpositive/llap/interval_3.q.out +++ b/ql/src/test/results/clientpositive/llap/interval_3.q.out @@ -159,10 +159,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: d1 - filterExpr: (((((((CAST( d_date AS DATE) + INTERVAL'1-0') + INTERVAL'0-2') + INTERVAL'5 00:00:00.000000000') + INTERVAL'0 04:00:00.000000000') + INTERVAL'0 00:10:00.000000000') + INTERVAL'0 00:00:09.000000000') is not null and (CAST( d_date AS DATE) + INTERVAL'1-2') is not null) (type: boolean) + filterExpr: CAST( d_date AS DATE) is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((((CAST( d_date AS DATE) + INTERVAL'1-0') + INTERVAL'0-2') + INTERVAL'5 00:00:00.000000000') + INTERVAL'0 04:00:00.000000000') + INTERVAL'0 00:10:00.000000000') + INTERVAL'0 00:00:09.000000000') is not null and (CAST( d_date AS DATE) + INTERVAL'1-2') is not null) (type: boolean) + predicate: CAST( d_date AS DATE) is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: d_week_seq (type: int), ((((((CAST( d_date AS DATE) + INTERVAL'1-0') + INTERVAL'0-2') + INTERVAL'5 00:00:00.000000000') + INTERVAL'0 04:00:00.000000000') + INTERVAL'0 00:10:00.000000000') + INTERVAL'0 00:00:09.000000000') (type: timestamp), (CAST( d_date AS DATE) + INTERVAL'1-2') (type: date) diff --git a/ql/src/test/results/clientpositive/llap/join13.q.out b/ql/src/test/results/clientpositive/llap/join13.q.out index 0bc700cd4561..dc395305938d 100644 --- a/ql/src/test/results/clientpositive/llap/join13.q.out +++ b/ql/src/test/results/clientpositive/llap/join13.q.out @@ -103,7 +103,6 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2 - residual filter predicates: {(UDFToDouble(_col2) + UDFToDouble(_col0)) is not null} Statistics: Num rows: 166 Data size: 43990 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: (UDFToDouble(_col2) + UDFToDouble(_col0)) (type: double) diff --git a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out index bfbc465981bc..f59a519ae615 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out @@ -3145,7 +3145,7 @@ STAGE PLANS: Statistics: Num rows: 16 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Filter Operator - predicate: (p_brand is not null and (UDFToDouble(p_type) + 2.0D) is not null) (type: boolean) + predicate: (p_brand is not null and UDFToDouble(p_type) is not null) (type: boolean) Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: (UDFToDouble(p_type) + 2.0D) (type: double), p_brand (type: string) diff --git a/ql/src/test/results/clientpositive/llap/vector_interval_mapjoin.q.out b/ql/src/test/results/clientpositive/llap/vector_interval_mapjoin.q.out index 76f4387f95ee..2bbf69ab5033 100644 --- a/ql/src/test/results/clientpositive/llap/vector_interval_mapjoin.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_interval_mapjoin.q.out @@ -206,7 +206,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: vectortab_a_1korc - filterExpr: ((dt - CAST( ts AS DATE)) is not null and s is not null) (type: boolean) + filterExpr: (dt is not null and CAST( ts AS DATE) is not null and s is not null) (type: boolean) Statistics: Num rows: 1000 Data size: 185480 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -214,18 +214,18 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: SelectColumnIsNotNull(col 16:interval_day_time)(children: DateColSubtractDateColumn(col 12:date, col 15:date)(children: CastTimestampToDate(col 10:timestamp) -> 15:date) -> 16:interval_day_time), SelectColumnIsNotNull(col 8:string)) - predicate: ((dt - CAST( ts AS DATE)) is not null and s is not null) (type: boolean) - Statistics: Num rows: 949 Data size: 176022 Basic stats: COMPLETE Column stats: COMPLETE + predicateExpression: FilterExprAndExpr(children: SelectColumnIsNotNull(col 12:date), SelectColumnIsNotNull(col 15:date)(children: CastTimestampToDate(col 10:timestamp) -> 15:date), SelectColumnIsNotNull(col 8:string)) + predicate: (dt is not null and CAST( ts AS DATE) is not null and s is not null) (type: boolean) + Statistics: Num rows: 905 Data size: 167854 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: s (type: string), (dt - CAST( ts AS DATE)) (type: interval_day_time) outputColumnNames: _col0, _col1 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [8, 18] - selectExpressions: DateColSubtractDateColumn(col 12:date, col 17:date)(children: CastTimestampToDate(col 10:timestamp) -> 17:date) -> 18:interval_day_time - Statistics: Num rows: 949 Data size: 100594 Basic stats: COMPLETE Column stats: COMPLETE + projectedOutputColumnNums: [8, 17] + selectExpressions: DateColSubtractDateColumn(col 12:date, col 16:date)(children: CastTimestampToDate(col 10:timestamp) -> 16:date) -> 17:interval_day_time + Statistics: Num rows: 905 Data size: 95930 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 @@ -240,21 +240,21 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 input vertices: 1 Map 2 - Statistics: Num rows: 943 Data size: 188600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 890 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col2 (type: string), _col1 (type: interval_day_time) outputColumnNames: _col0, _col1, _col2 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [8, 8, 18] - Statistics: Num rows: 943 Data size: 188600 Basic stats: COMPLETE Column stats: COMPLETE + projectedOutputColumnNums: [8, 8, 17] + Statistics: Num rows: 890 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 943 Data size: 188600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 890 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -274,7 +274,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: vectortab_b_1korc - filterExpr: ((dt - CAST( ts AS DATE)) is not null and s is not null) (type: boolean) + filterExpr: (dt is not null and CAST( ts AS DATE) is not null and s is not null) (type: boolean) Statistics: Num rows: 1000 Data size: 184664 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -282,18 +282,18 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: SelectColumnIsNotNull(col 16:interval_day_time)(children: DateColSubtractDateColumn(col 12:date, col 15:date)(children: CastTimestampToDate(col 10:timestamp) -> 15:date) -> 16:interval_day_time), SelectColumnIsNotNull(col 8:string)) - predicate: ((dt - CAST( ts AS DATE)) is not null and s is not null) (type: boolean) - Statistics: Num rows: 943 Data size: 174122 Basic stats: COMPLETE Column stats: COMPLETE + predicateExpression: FilterExprAndExpr(children: SelectColumnIsNotNull(col 12:date), SelectColumnIsNotNull(col 15:date)(children: CastTimestampToDate(col 10:timestamp) -> 15:date), SelectColumnIsNotNull(col 8:string)) + predicate: (dt is not null and CAST( ts AS DATE) is not null and s is not null) (type: boolean) + Statistics: Num rows: 890 Data size: 164340 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: s (type: string), (dt - CAST( ts AS DATE)) (type: interval_day_time) outputColumnNames: _col0, _col1 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [8, 18] - selectExpressions: DateColSubtractDateColumn(col 12:date, col 17:date)(children: CastTimestampToDate(col 10:timestamp) -> 17:date) -> 18:interval_day_time - Statistics: Num rows: 943 Data size: 99958 Basic stats: COMPLETE Column stats: COMPLETE + projectedOutputColumnNums: [8, 17] + selectExpressions: DateColSubtractDateColumn(col 12:date, col 16:date)(children: CastTimestampToDate(col 10:timestamp) -> 16:date) -> 17:interval_day_time + Statistics: Num rows: 890 Data size: 94340 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: interval_day_time) null sort order: zz @@ -303,7 +303,7 @@ STAGE PLANS: className: VectorReduceSinkMultiKeyOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 943 Data size: 99958 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 890 Data size: 94340 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: From 7ec2e5987b912b2109f964c2778d0299702ef59f Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Tue, 1 Sep 2026 17:25:09 +0100 Subject: [PATCH 07/24] Adjust test files that suffered from "filter split" on the last upgrade (coming back to their original pre-1.42 state) --- .../clientpositive/llap/auto_join2.q.out | 106 ++++++------ .../clientpositive/llap/auto_join_stats.q.out | 76 ++++---- .../llap/auto_join_stats2.q.out | 162 ++++++++---------- .../results/clientpositive/llap/join2.q.out | 38 ++-- 4 files changed, 173 insertions(+), 209 deletions(-) diff --git a/ql/src/test/results/clientpositive/llap/auto_join2.q.out b/ql/src/test/results/clientpositive/llap/auto_join2.q.out index bc77780295e2..4ff363c6a7f1 100644 --- a/ql/src/test/results/clientpositive/llap/auto_join2.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_join2.q.out @@ -38,73 +38,70 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src1 - filterExpr: key is not null (type: boolean) - probeDecodeDetails: cacheKey:HASH_MAP_MAPJOIN_49_container, bigKeyColName:key, smallTablePos:1, keyRatio:0.632 + filterExpr: (key is not null and UDFToDouble(key) is not null) (type: boolean) + probeDecodeDetails: cacheKey:HASH_MAP_MAPJOIN_45_container, bigKeyColName:key, smallTablePos:1, keyRatio:0.632 Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: key is not null (type: boolean) + predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), UDFToDouble(key) (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col1 is not null (type: boolean) - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Reducer 4 + Statistics: Num rows: 791 Data size: 81473 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3 + 0 (_col1 + _col3) (type: double) + 1 _col1 (type: double) + outputColumnNames: _col0, _col4 input vertices: - 1 Reducer 4 - Statistics: Num rows: 791 Data size: 81473 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 (_col1 + _col3) (type: double) - 1 _col1 (type: double) - outputColumnNames: _col0, _col4 - input vertices: - 1 Map 3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col4 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j2 Select Operator - expressions: UDFToInteger(_col0) (type: int), _col4 (type: string) - outputColumnNames: _col0, _col1 + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j2 - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(key), max(key), count(1), count(key), compute_bit_vector_hll(key), max(length(value)), avg(COALESCE(length(value),0)), count(value), compute_bit_vector_hll(value) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Group By Operator + aggregations: min(key), max(key), count(1), count(key), compute_bit_vector_hll(key), max(length(value)), avg(COALESCE(length(value),0)), count(value), compute_bit_vector_hll(value) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: Statistics: Num rows: 1 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: struct), _col7 (type: bigint), _col8 (type: binary) + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: struct), _col7 (type: bigint), _col8 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs Map 3 Map Operator Tree: TableScan alias: src3 - filterExpr: (UDFToDouble(key) is not null or key is not null) (type: boolean) + filterExpr: (UDFToDouble(key) is not null or (key is not null and UDFToDouble(key) is not null)) (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: UDFToDouble(key) is not null (type: boolean) @@ -121,22 +118,19 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) Filter Operator - predicate: key is not null (type: boolean) + predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), UDFToDouble(key) (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col1 is not null (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) + value expressions: _col1 (type: double) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/auto_join_stats.q.out b/ql/src/test/results/clientpositive/llap/auto_join_stats.q.out index e4c590dda7e5..8ed693529f7c 100644 --- a/ql/src/test/results/clientpositive/llap/auto_join_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_join_stats.q.out @@ -48,50 +48,44 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src1 - filterExpr: key is not null (type: boolean) + filterExpr: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: key is not null (type: boolean) + predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), UDFToDouble(key) (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col1 is not null (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) + value expressions: _col1 (type: double) Execution mode: vectorized, llap LLAP IO: all inputs Map 3 Map Operator Tree: TableScan alias: src2 - filterExpr: key is not null (type: boolean) + filterExpr: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: key is not null (type: boolean) + predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), UDFToDouble(key) (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col1 is not null (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) + value expressions: _col1 (type: double) Execution mode: vectorized, llap LLAP IO: all inputs Map 4 @@ -219,50 +213,44 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src1 - filterExpr: key is not null (type: boolean) + filterExpr: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: key is not null (type: boolean) + predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), UDFToDouble(key) (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col1 is not null (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) + value expressions: _col1 (type: double) Execution mode: vectorized, llap LLAP IO: all inputs Map 3 Map Operator Tree: TableScan alias: src2 - filterExpr: key is not null (type: boolean) + filterExpr: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: key is not null (type: boolean) + predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), UDFToDouble(key) (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col1 is not null (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) + value expressions: _col1 (type: double) Execution mode: vectorized, llap LLAP IO: all inputs Map 4 diff --git a/ql/src/test/results/clientpositive/llap/auto_join_stats2.q.out b/ql/src/test/results/clientpositive/llap/auto_join_stats2.q.out index 6b775c286e7b..f1a52ab82b0c 100644 --- a/ql/src/test/results/clientpositive/llap/auto_join_stats2.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_join_stats2.q.out @@ -14,7 +14,7 @@ POSTHOOK: query: load data local inpath '../../data/files/T1.txt' into table sma POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smalltable -Warning: Map Join MAPJOIN[37][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[33][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: explain select src1.key, src2.key, smalltable.key from src src1 JOIN src src2 ON (src1.key = src2.key) JOIN smalltable ON (src1.key + src2.key = smalltable.key) PREHOOK: type: QUERY PREHOOK: Input: default@smalltable @@ -41,50 +41,47 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src2 - filterExpr: key is not null (type: boolean) + filterExpr: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: key is not null (type: boolean) + predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), UDFToDouble(key) (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col1 is not null (type: boolean) - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3 + input vertices: + 1 Map 2 + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3 + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - input vertices: - 1 Map 3 - residual filter predicates: {((_col5 + _col1) = _col3)} + 1 Map 3 + residual filter predicates: {((_col5 + _col1) = _col3)} + Statistics: Num rows: 275 Data size: 77000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col4 (type: string), _col0 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 275 Data size: 77000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col4 (type: string), _col0 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false Statistics: Num rows: 275 Data size: 77000 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 275 Data size: 77000 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: all inputs Map 2 @@ -111,25 +108,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src1 - filterExpr: key is not null (type: boolean) + filterExpr: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: key is not null (type: boolean) + predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), UDFToDouble(key) (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col1 is not null (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) + value expressions: _col1 (type: double) Execution mode: vectorized, llap LLAP IO: all inputs @@ -139,7 +133,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[37][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[33][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: select src1.key, src2.key, smalltable.key from src src1 JOIN src src2 ON (src1.key = src2.key) JOIN smalltable ON (src1.key + src2.key = smalltable.key) PREHOOK: type: QUERY PREHOOK: Input: default@smalltable @@ -204,85 +198,79 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src1 - filterExpr: key is not null (type: boolean) - probeDecodeDetails: cacheKey:HASH_MAP_MAPJOIN_55_container, bigKeyColName:key, smallTablePos:1, keyRatio:0.632 + filterExpr: (key is not null and UDFToDouble(key) is not null) (type: boolean) + probeDecodeDetails: cacheKey:HASH_MAP_MAPJOIN_51_container, bigKeyColName:key, smallTablePos:1, keyRatio:0.632 Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: key is not null (type: boolean) + predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), UDFToDouble(key) (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col1 is not null (type: boolean) - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + input vertices: + 1 Map 2 + Statistics: Num rows: 791 Data size: 150290 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + 0 (_col1 + _col3) (type: double) + 1 _col0 (type: double) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 - Statistics: Num rows: 791 Data size: 150290 Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 3 + Statistics: Num rows: 870 Data size: 165319 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 (_col1 + _col3) (type: double) - 1 _col0 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 + 1 _col1 (type: double) + outputColumnNames: _col0, _col2, _col5 input vertices: - 1 Map 3 - Statistics: Num rows: 870 Data size: 165319 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 (_col1 + _col3) (type: double) - 1 _col1 (type: double) - outputColumnNames: _col0, _col2, _col5 - input vertices: - 1 Map 4 + 1 Map 4 + Statistics: Num rows: 957 Data size: 181850 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col2 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 957 Data size: 181850 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col2 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false Statistics: Num rows: 957 Data size: 181850 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 957 Data size: 181850 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs Map 2 Map Operator Tree: TableScan alias: src2 - filterExpr: key is not null (type: boolean) + filterExpr: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: key is not null (type: boolean) + predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), UDFToDouble(key) (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col1 is not null (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) + value expressions: _col1 (type: double) Execution mode: vectorized, llap LLAP IO: all inputs Map 3 diff --git a/ql/src/test/results/clientpositive/llap/join2.q.out b/ql/src/test/results/clientpositive/llap/join2.q.out index 32912e401227..3b46b489faa3 100644 --- a/ql/src/test/results/clientpositive/llap/join2.q.out +++ b/ql/src/test/results/clientpositive/llap/join2.q.out @@ -37,32 +37,29 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src1 - filterExpr: key is not null (type: boolean) + filterExpr: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: key is not null (type: boolean) + predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), UDFToDouble(key) (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col1 is not null (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) + value expressions: _col1 (type: double) Execution mode: vectorized, llap LLAP IO: all inputs Map 4 Map Operator Tree: TableScan alias: src3 - filterExpr: (UDFToDouble(key) is not null or key is not null) (type: boolean) + filterExpr: (UDFToDouble(key) is not null or (key is not null and UDFToDouble(key) is not null)) (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: UDFToDouble(key) is not null (type: boolean) @@ -79,22 +76,19 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) Filter Operator - predicate: key is not null (type: boolean) + predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), UDFToDouble(key) (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col1 is not null (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) + value expressions: _col1 (type: double) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 From aab0e088cd0763974560055b35f6e46692c78962 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Tue, 1 Sep 2026 17:27:03 +0100 Subject: [PATCH 08/24] Adjust test files 'ROW__ID.writeid > 1L' ==> '(ROW__ID.writeid <= 1L) is not true' due to CALCITE-7636 --- .../llap/materialized_view_cluster.q.out | 34 ++++++------ .../materialized_view_create_rewrite_3.q.out | 4 +- .../materialized_view_create_rewrite_4.q.out | 8 +-- .../materialized_view_create_rewrite_5.q.out | 24 ++++----- .../materialized_view_create_rewrite_7.q.out | 30 +++++------ .../materialized_view_create_rewrite_8.q.out | 20 +++---- ...terialized_view_create_rewrite_nulls.q.out | 30 +++++------ ...ized_view_create_rewrite_one_key_gby.q.out | 4 +- ...ed_view_create_rewrite_rebuild_dummy.q.out | 4 +- ...ized_view_create_rewrite_time_window.q.out | 4 +- .../materialized_view_distribute_sort.q.out | 34 ++++++------ .../llap/materialized_view_parquet.q.out | 14 ++--- .../materialized_view_partition_cluster.q.out | 50 ++++++++--------- .../llap/materialized_view_partitioned.q.out | 54 +++++++++---------- ..._view_partitioned_create_rewrite_agg.q.out | 48 ++++++++--------- ...iew_partitioned_create_rewrite_agg_2.q.out | 14 ++--- ...iew_partitioned_create_rewrite_agg_3.q.out | 48 ++++++++--------- .../llap/materialized_view_rebuild_2.q.out | 4 +- .../llap/materialized_view_rebuild_3.q.out | 18 +++---- ...sketches_materialized_view_cume_dist.q.out | 10 ++-- .../sketches_materialized_view_ntile.q.out | 10 ++-- ...es_materialized_view_percentile_disc.q.out | 10 ++-- .../sketches_materialized_view_rank.q.out | 10 ++-- .../sketches_materialized_view_rollup2.q.out | 10 ++-- 24 files changed, 248 insertions(+), 248 deletions(-) diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_cluster.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_cluster.q.out index 2f57559da121..afb6063bf899 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_cluster.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_cluster.q.out @@ -490,18 +490,18 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src_txn - filterExpr: ((ROW__ID.writeid > 1L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D) and (ROW__ID.writeid <= 1L) is not true) (type: boolean) Statistics: Num rows: 501 Data size: 90180 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((ROW__ID.writeid > 1L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) - Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D) and (ROW__ID.writeid <= 1L) is not true) (type: boolean) + Statistics: Num rows: 27 Data size: 4860 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: value (type: string), key (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4860 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4860 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -510,10 +510,10 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: value, key - Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4860 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(value)), avg(COALESCE(length(value),0)), count(1), count(value), compute_bit_vector_hll(value), max(length(key)), avg(COALESCE(length(key),0)), count(key), compute_bit_vector_hll(key) - minReductionHashAggr: 0.9444444 + minReductionHashAggr: 0.962963 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 472 Basic stats: COMPLETE Column stats: COMPLETE @@ -796,21 +796,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src_txn - filterExpr: ((ROW__ID.writeid > 2L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D) and (ROW__ID.writeid <= 2L) is not true) (type: boolean) Statistics: Num rows: 502 Data size: 90862 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((ROW__ID.writeid > 2L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D) and (ROW__ID.writeid <= 2L) is not true) (type: boolean) + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) @@ -845,14 +845,14 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -861,10 +861,10 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: value, key - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(value)), avg(COALESCE(length(value),0)), count(1), count(value), compute_bit_vector_hll(value), max(length(key)), avg(COALESCE(length(key),0)), count(key), compute_bit_vector_hll(key) - minReductionHashAggr: 0.9444444 + minReductionHashAggr: 0.962963 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 472 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out index e0718e569551..8b820b73a3db 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out @@ -542,10 +542,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cmv_basetable_2 - filterExpr: ((ROW__ID.writeid > 1L) and (c > 10) and a is not null) (type: boolean) + filterExpr: ((c > 10) and (ROW__ID.writeid <= 1L) is not true and a is not null) (type: boolean) Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((ROW__ID.writeid > 1L) and (c > 10) and a is not null) (type: boolean) + predicate: ((c > 10) and (ROW__ID.writeid <= 1L) is not true and a is not null) (type: boolean) Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: decimal(10,2)) diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out index 3b05f9b86341..b439c1f21135 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out @@ -801,10 +801,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cmv_basetable_2_n2 - filterExpr: ((ROW__ID.writeid > 1L) and (c > 10) and a is not null) (type: boolean) + filterExpr: ((c > 10) and (ROW__ID.writeid <= 1L) is not true and a is not null) (type: boolean) Statistics: Num rows: 3 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((ROW__ID.writeid > 1L) and (c > 10) and a is not null) (type: boolean) + predicate: ((c > 10) and (ROW__ID.writeid <= 1L) is not true and a is not null) (type: boolean) Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: decimal(10,2)), d (type: int) @@ -1913,10 +1913,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cmv_basetable_2_n2 - filterExpr: ((ROW__ID.writeid > 4L) and (c > 10) and a is not null) (type: boolean) + filterExpr: ((c > 10) and (ROW__ID.writeid <= 4L) is not true and a is not null) (type: boolean) Statistics: Num rows: 3 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((ROW__ID.writeid > 4L) and (c > 10) and a is not null) (type: boolean) + predicate: ((c > 10) and (ROW__ID.writeid <= 4L) is not true and a is not null) (type: boolean) Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: decimal(10,2)), d (type: int) diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out index bb85ddc371db..1dddc405cff5 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out @@ -266,21 +266,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cmv_basetable_2_n3 - filterExpr: ((ROW__ID.writeid > 1L) and (c > 10) and a is not null) (type: boolean) + filterExpr: ((c > 10) and (ROW__ID.writeid <= 1L) is not true and a is not null) (type: boolean) Statistics: Num rows: 4 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((ROW__ID.writeid > 1L) and (c > 10) and a is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((c > 10) and (ROW__ID.writeid <= 1L) is not true and a is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: decimal(10,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: decimal(10,2)) Execution mode: llap LLAP IO: may be used (ACID table) @@ -294,14 +294,14 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col2 - Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col2 (type: decimal(10,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -311,10 +311,10 @@ STAGE PLANS: Select Operator expressions: _col0 (type: int), _col1 (type: decimal(10,2)) outputColumnNames: a, c - Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(a), max(a), count(1), count(a), compute_bit_vector_hll(a), min(c), max(c), count(c), compute_bit_vector_hll(c) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.75 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 544 Basic stats: COMPLETE Column stats: COMPLETE @@ -1037,10 +1037,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cmv_basetable_2_n3 - filterExpr: ((ROW__ID.writeid > 4L) and (c > 10) and a is not null) (type: boolean) + filterExpr: ((c > 10) and (ROW__ID.writeid <= 4L) is not true and a is not null) (type: boolean) Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((ROW__ID.writeid > 4L) and (c > 10) and a is not null) (type: boolean) + predicate: ((c > 10) and (ROW__ID.writeid <= 4L) is not true and a is not null) (type: boolean) Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: decimal(10,2)) diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_7.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_7.q.out index 7c3ba78c396f..e74fcae7002a 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_7.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_7.q.out @@ -108,7 +108,7 @@ HiveProject(t1.a=[$3], _c1=[CASE(IS NULL($1), $4, IS NULL($4), $1, +($4, $1))]) HiveTableScan(table=[[default, mat1]], table:alias=[default.mat1]) HiveProject(a=[$0], $f1=[$1]) HiveAggregate(group=[{0}], agg#0=[count()]) - HiveFilter(condition=[<(2, $5.writeid)]) + HiveFilter(condition=[IS NOT TRUE(>=(2, $5.writeid))]) HiveTableScan(table=[[default, t1]], table:alias=[t1]) PREHOOK: query: explain @@ -170,28 +170,28 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1 - filterExpr: (ROW__ID.writeid > 2L) (type: boolean) + filterExpr: (ROW__ID.writeid <= 2L) is not true (type: boolean) Statistics: Num rows: 11 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (ROW__ID.writeid > 2L) (type: boolean) - Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (ROW__ID.writeid <= 2L) is not true (type: boolean) + Statistics: Num rows: 5 Data size: 450 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: char(15)) outputColumnNames: a - Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 450 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() keys: a (type: char(15)) minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 294 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: char(15)) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: char(15)) - Statistics: Num rows: 3 Data size: 294 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) @@ -206,7 +206,7 @@ STAGE PLANS: 1 _col0 (type: char(15)) nullSafes: [true] outputColumnNames: _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 744 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1028 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col2 (type: boolean) Statistics: Num rows: 1 Data size: 186 Basic stats: COMPLETE Column stats: COMPLETE @@ -253,14 +253,14 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary) Filter Operator predicate: _col2 is null (type: boolean) - Statistics: Num rows: 1 Data size: 186 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col4 (type: char(15)), CASE WHEN (_col1 is null) THEN (_col5) WHEN (_col5 is null) THEN (_col1) ELSE ((_col5 + _col1)) END (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -270,10 +270,10 @@ STAGE PLANS: Select Operator expressions: _col0 (type: char(15)), _col1 (type: bigint) outputColumnNames: a, _c1 - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(a)), avg(COALESCE(length(a),0)), count(1), count(a), compute_bit_vector_hll(a), min(_c1), max(_c1), count(_c1), compute_bit_vector_hll(_c1) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE @@ -344,13 +344,13 @@ STAGE PLANS: keys: KEY._col0 (type: char(15)) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 294 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: char(15)) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: char(15)) - Statistics: Num rows: 3 Data size: 294 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) Stage: Stage-4 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_8.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_8.q.out index 68eba9e63600..81e00f3b3971 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_8.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_8.q.out @@ -197,23 +197,23 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cmv_basetable_2_n3 - filterExpr: ((ROW__ID.writeid > 1L) and (c > 10) and a is not null) (type: boolean) + filterExpr: ((c > 10) and (ROW__ID.writeid <= 1L) is not true and a is not null) (type: boolean) properties: insertonly.fetch.bucketid TRUE Statistics: Num rows: 4 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((ROW__ID.writeid > 1L) and (c > 10) and a is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((c > 10) and (ROW__ID.writeid <= 1L) is not true and a is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: decimal(10,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: decimal(10,2)) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) @@ -227,14 +227,14 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col2 - Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col2 (type: decimal(10,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -244,10 +244,10 @@ STAGE PLANS: Select Operator expressions: _col0 (type: int), _col1 (type: decimal(10,2)) outputColumnNames: a, c - Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(a), max(a), count(1), count(a), compute_bit_vector_hll(a), min(c), max(c), count(c), compute_bit_vector_hll(c) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.75 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 544 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_nulls.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_nulls.q.out index adf42999c5f7..a06845287444 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_nulls.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_nulls.q.out @@ -123,7 +123,7 @@ HiveProject(t1.a=[$6], t1.b=[$7], _c2=[CASE(IS NULL($2), $8, IS NULL($8), $2, +( HiveTableScan(table=[[default, mat1]], table:alias=[default.mat1]) HiveProject(a=[$0], b=[$1], $f2=[$2], $f3=[$3], $f4=[$4]) HiveAggregate(group=[{0, 1}], agg#0=[sum($3)], agg#1=[min($3)], agg#2=[max($3)]) - HiveFilter(condition=[AND(<(1, $6.writeid), >($2, 10:DECIMAL(2, 0)))]) + HiveFilter(condition=[AND(>($2, 10:DECIMAL(2, 0)), IS NOT TRUE(>=(1, $6.writeid)))]) HiveTableScan(table=[[default, t1]], table:alias=[t1]) PREHOOK: query: explain @@ -186,28 +186,28 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1 - filterExpr: ((ROW__ID.writeid > 1L) and (c > 10)) (type: boolean) + filterExpr: ((c > 10) and (ROW__ID.writeid <= 1L) is not true) (type: boolean) Statistics: Num rows: 22 Data size: 4986 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((ROW__ID.writeid > 1L) and (c > 10)) (type: boolean) - Statistics: Num rows: 7 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((c > 10) and (ROW__ID.writeid <= 1L) is not true) (type: boolean) + Statistics: Num rows: 11 Data size: 2554 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), b (type: varchar(256)), d (type: int) outputColumnNames: a, b, d - Statistics: Num rows: 7 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 2554 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(d), min(d), max(d) keys: a (type: int), b (type: varchar(256)) minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 7 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 1462 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: varchar(256)) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: varchar(256)) - Statistics: Num rows: 7 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 1462 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: int), _col4 (type: int) Execution mode: llap LLAP IO: may be used (ACID table) @@ -222,7 +222,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: varchar(256)) nullSafes: [true, true] outputColumnNames: _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 10 Data size: 2052 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 3580 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col5 (type: boolean) Statistics: Num rows: 1 Data size: 230 Basic stats: COMPLETE Column stats: COMPLETE @@ -269,14 +269,14 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: struct), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: int), _col15 (type: bigint), _col16 (type: binary), _col17 (type: int), _col18 (type: int), _col19 (type: bigint), _col20 (type: binary) Filter Operator predicate: _col5 is null (type: boolean) - Statistics: Num rows: 3 Data size: 674 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 2014 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col7 (type: int), _col8 (type: varchar(256)), CASE WHEN (_col2 is null) THEN (_col9) WHEN (_col9 is null) THEN (_col2) ELSE ((_col9 + _col2)) END (type: bigint), CASE WHEN (_col3 is null) THEN (_col10) WHEN (_col10 is null) THEN (_col3) ELSE (if((_col10 < _col3), _col10, _col3)) END (type: int), CASE WHEN (_col4 is null) THEN (_col11) WHEN (_col11 is null) THEN (_col4) ELSE (if((_col11 > _col4), _col11, _col4)) END (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 3 Data size: 386 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 1358 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 386 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 1358 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -286,10 +286,10 @@ STAGE PLANS: Select Operator expressions: _col0 (type: int), _col1 (type: varchar(256)), _col2 (type: bigint), _col3 (type: int), _col4 (type: int) outputColumnNames: a, b, _c2, _c3, _c4 - Statistics: Num rows: 3 Data size: 386 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 1358 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(a), max(a), count(1), count(a), compute_bit_vector_hll(a), max(length(b)), avg(COALESCE(length(b),0)), count(b), compute_bit_vector_hll(b), min(_c2), max(_c2), count(_c2), compute_bit_vector_hll(_c2), min(_c3), max(_c3), count(_c3), compute_bit_vector_hll(_c3), min(_c4), max(_c4), count(_c4), compute_bit_vector_hll(_c4) - minReductionHashAggr: 0.6666666 + minReductionHashAggr: 0.9090909 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE @@ -360,13 +360,13 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: varchar(256)) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 7 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 1462 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: varchar(256)) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: varchar(256)) - Statistics: Num rows: 7 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 1462 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: int), _col4 (type: int) Stage: Stage-4 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_one_key_gby.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_one_key_gby.q.out index 25fe6711ccc5..c869be4b56e9 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_one_key_gby.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_one_key_gby.q.out @@ -171,10 +171,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cmv_basetable_2_n2 - filterExpr: ((ROW__ID.writeid > 1L) and (c > 10) and a is not null) (type: boolean) + filterExpr: ((c > 10) and (ROW__ID.writeid <= 1L) is not true and a is not null) (type: boolean) Statistics: Num rows: 3 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((ROW__ID.writeid > 1L) and (c > 10) and a is not null) (type: boolean) + predicate: ((c > 10) and (ROW__ID.writeid <= 1L) is not true and a is not null) (type: boolean) Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), d (type: int) diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out index 7e951ad1c638..246a0f48163b 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out @@ -542,10 +542,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cmv_basetable_2_n0 - filterExpr: ((ROW__ID.writeid > 1L) and (c > 10) and a is not null) (type: boolean) + filterExpr: ((c > 10) and (ROW__ID.writeid <= 1L) is not true and a is not null) (type: boolean) Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((ROW__ID.writeid > 1L) and (c > 10) and a is not null) (type: boolean) + predicate: ((c > 10) and (ROW__ID.writeid <= 1L) is not true and a is not null) (type: boolean) Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: decimal(10,2)) diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out index 4ebaf7f33a97..6703c4c0b044 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out @@ -676,10 +676,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cmv_basetable_2_n1 - filterExpr: ((ROW__ID.writeid > 1L) and (c > 10) and a is not null) (type: boolean) + filterExpr: ((c > 10) and (ROW__ID.writeid <= 1L) is not true and a is not null) (type: boolean) Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((ROW__ID.writeid > 1L) and (c > 10) and a is not null) (type: boolean) + predicate: ((c > 10) and (ROW__ID.writeid <= 1L) is not true and a is not null) (type: boolean) Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: decimal(10,2)) diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_distribute_sort.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_distribute_sort.q.out index c86001ada212..26c15292f9f6 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_distribute_sort.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_distribute_sort.q.out @@ -672,18 +672,18 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src_txn - filterExpr: ((ROW__ID.writeid > 1L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D) and (ROW__ID.writeid <= 1L) is not true) (type: boolean) Statistics: Num rows: 501 Data size: 90180 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((ROW__ID.writeid > 1L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) - Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D) and (ROW__ID.writeid <= 1L) is not true) (type: boolean) + Statistics: Num rows: 27 Data size: 4860 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: value (type: string), key (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4860 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4860 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -692,10 +692,10 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: value, key - Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4860 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(value)), avg(COALESCE(length(value),0)), count(1), count(value), compute_bit_vector_hll(value), max(length(key)), avg(COALESCE(length(key),0)), count(key), compute_bit_vector_hll(key) - minReductionHashAggr: 0.9444444 + minReductionHashAggr: 0.962963 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 472 Basic stats: COMPLETE Column stats: COMPLETE @@ -1219,21 +1219,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src_txn - filterExpr: ((ROW__ID.writeid > 2L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D) and (ROW__ID.writeid <= 2L) is not true) (type: boolean) Statistics: Num rows: 502 Data size: 90862 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((ROW__ID.writeid > 2L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D) and (ROW__ID.writeid <= 2L) is not true) (type: boolean) + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) @@ -1268,14 +1268,14 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1284,10 +1284,10 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: value, key - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(value)), avg(COALESCE(length(value),0)), count(1), count(value), compute_bit_vector_hll(value), max(length(key)), avg(COALESCE(length(key),0)), count(key), compute_bit_vector_hll(key) - minReductionHashAggr: 0.9444444 + minReductionHashAggr: 0.962963 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 472 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_parquet.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_parquet.q.out index 84c858d69f7b..dabe42d7828e 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_parquet.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_parquet.q.out @@ -849,30 +849,30 @@ STAGE PLANS: Map Operator Tree: TableScan alias: emps_parquet_n3 - filterExpr: (ROW__ID.writeid > 3L) (type: boolean) + filterExpr: (ROW__ID.writeid <= 3L) is not true (type: boolean) properties: insertonly.fetch.bucketid TRUE Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (ROW__ID.writeid > 3L) (type: boolean) - Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (ROW__ID.writeid <= 3L) is not true (type: boolean) + Statistics: Num rows: 3 Data size: 294 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: empid (type: int), name (type: varchar(256)), salary (type: float) outputColumnNames: empid, name, salary - Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 294 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), sum(empid) keys: name (type: varchar(256)), salary (type: float) minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 330 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: varchar(256)), _col1 (type: float) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: varchar(256)), _col1 (type: float) - Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 330 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs (cache only) @@ -909,7 +909,7 @@ STAGE PLANS: keys: KEY._col0 (type: varchar(256)), KEY._col1 (type: float) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 330 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col2), sum(_col3) keys: _col0 (type: varchar(256)), _col1 (type: float) diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_partition_cluster.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_partition_cluster.q.out index 2a59f7983fa3..2f09d060782c 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_partition_cluster.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_partition_cluster.q.out @@ -978,18 +978,18 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src_txn - filterExpr: ((ROW__ID.writeid > 1L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D) and (ROW__ID.writeid <= 1L) is not true) (type: boolean) Statistics: Num rows: 501 Data size: 90180 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((ROW__ID.writeid > 1L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) - Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D) and (ROW__ID.writeid <= 1L) is not true) (type: boolean) + Statistics: Num rows: 27 Data size: 4860 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: value (type: string), key (type: string), (UDFToDouble(key) + 100.0D) (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 18 Data size: 3384 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 5076 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 18 Data size: 3384 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 5076 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -998,20 +998,20 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) outputColumnNames: value, key, partkey - Statistics: Num rows: 18 Data size: 3384 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 5076 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(value)), avg(COALESCE(length(value),0)), count(1), count(value), compute_bit_vector_hll(value), max(length(key)), avg(COALESCE(length(key),0)), count(key), compute_bit_vector_hll(key) keys: partkey (type: double) minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 18 Data size: 8640 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 12960 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: double) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 18 Data size: 8640 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 12960 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: struct), _col8 (type: bigint), _col9 (type: binary) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) @@ -1023,14 +1023,14 @@ STAGE PLANS: keys: KEY._col0 (type: double) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 18 Data size: 6192 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 9288 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col6,0)) (type: bigint), COALESCE(_col7,0) (type: double), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), _col0 (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 18 Data size: 9720 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 14580 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 18 Data size: 9720 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 14580 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1400,21 +1400,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src_txn - filterExpr: ((ROW__ID.writeid > 2L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D) and (ROW__ID.writeid <= 2L) is not true) (type: boolean) Statistics: Num rows: 502 Data size: 90862 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((ROW__ID.writeid > 2L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D) and (ROW__ID.writeid <= 2L) is not true) (type: boolean) + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string), (UDFToDouble(key) + 100.0D) (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 18 Data size: 3402 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 5103 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 18 Data size: 3402 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 5103 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: double) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) @@ -1449,14 +1449,14 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 18 Data size: 3402 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 5103 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: string), _col2 (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 18 Data size: 3402 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 5103 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 18 Data size: 3402 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 5103 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1465,20 +1465,20 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) outputColumnNames: value, key, partkey - Statistics: Num rows: 18 Data size: 3402 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 5103 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(value)), avg(COALESCE(length(value),0)), count(1), count(value), compute_bit_vector_hll(value), max(length(key)), avg(COALESCE(length(key),0)), count(key), compute_bit_vector_hll(key) keys: partkey (type: double) minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 18 Data size: 8640 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 12960 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: double) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 18 Data size: 8640 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 12960 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: struct), _col8 (type: bigint), _col9 (type: binary) Reducer 3 Execution mode: vectorized, llap @@ -1488,14 +1488,14 @@ STAGE PLANS: keys: KEY._col0 (type: double) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 18 Data size: 6192 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 9288 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col6,0)) (type: bigint), COALESCE(_col7,0) (type: double), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), _col0 (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 18 Data size: 9720 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 14580 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 18 Data size: 9720 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 14580 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out index b264b6acbd9a..c57c4817c7c8 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out @@ -652,39 +652,39 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src_txn - filterExpr: ((ROW__ID.writeid > 1L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D) and (ROW__ID.writeid <= 1L) is not true) (type: boolean) Statistics: Num rows: 501 Data size: 90180 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((ROW__ID.writeid > 1L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) - Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D) and (ROW__ID.writeid <= 1L) is not true) (type: boolean) + Statistics: Num rows: 27 Data size: 4860 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: value (type: string), key (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4860 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4860 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: value, key - Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4860 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(value)), avg(COALESCE(length(value),0)), count(1), count(value), compute_bit_vector_hll(value) keys: key (type: string) minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 18 Data size: 5886 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 8829 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 18 Data size: 5886 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 8829 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) @@ -697,7 +697,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4860 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -711,14 +711,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 18 Data size: 4662 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 6993 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 18 Data size: 6354 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 9531 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 18 Data size: 6354 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 9531 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -991,21 +991,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src_txn - filterExpr: ((ROW__ID.writeid > 2L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D) and (ROW__ID.writeid <= 2L) is not true) (type: boolean) Statistics: Num rows: 502 Data size: 90862 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((ROW__ID.writeid > 2L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D) and (ROW__ID.writeid <= 2L) is not true) (type: boolean) + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) @@ -1040,35 +1040,35 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: value, key - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(value)), avg(COALESCE(length(value),0)), count(1), count(value), compute_bit_vector_hll(value) keys: key (type: string) minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 18 Data size: 5886 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 8829 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 18 Data size: 5886 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 8829 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary) Reducer 3 Execution mode: vectorized, llap @@ -1079,7 +1079,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4887 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1093,14 +1093,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 18 Data size: 4662 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 6993 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 18 Data size: 6354 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 9531 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 18 Data size: 6354 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 9531 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_create_rewrite_agg.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_create_rewrite_agg.q.out index 54aa1228203c..8a32a9321f76 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_create_rewrite_agg.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_create_rewrite_agg.q.out @@ -136,11 +136,11 @@ HiveProject(b=[$0], sumc=[$2], a=[$1]) HiveTableScan(table=[[default, mat1]], table:alias=[default.mat1]) HiveProject(b=[$1], a=[$0]) HiveAggregate(group=[{0, 1}]) - HiveFilter(condition=[<(1, $5.writeid)]) + HiveFilter(condition=[IS NOT TRUE(>=(1, $5.writeid))]) HiveTableScan(table=[[default, t1]], table:alias=[t1]) HiveProject(b=[$1], a=[$0], $f2=[$2]) HiveAggregate(group=[{0, 1}], agg#0=[sum($2)]) - HiveFilter(condition=[<(1, $5.writeid)]) + HiveFilter(condition=[IS NOT TRUE(>=(1, $5.writeid))]) HiveTableScan(table=[[default, t1]], table:alias=[t1]) PREHOOK: query: EXPLAIN @@ -202,44 +202,44 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1 - filterExpr: (ROW__ID.writeid > 1L) (type: boolean) + filterExpr: (ROW__ID.writeid <= 1L) is not true (type: boolean) Statistics: Num rows: 10 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (ROW__ID.writeid > 1L) (type: boolean) - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (ROW__ID.writeid <= 1L) is not true (type: boolean) + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), b (type: int) outputColumnNames: a, b - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: a (type: int), b (type: int) minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), b (type: int), c (type: int) outputColumnNames: a, b, c - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(c) keys: b (type: int), a (type: int) minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) @@ -258,16 +258,16 @@ STAGE PLANS: Group By Operator aggregations: sum(_col2) keys: _col0 (type: int), _col1 (type: int) - minReductionHashAggr: 0.4285714 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) Reducer 4 Execution mode: vectorized, llap @@ -277,14 +277,14 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col2 (type: bigint), _col1 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -294,7 +294,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: int) outputColumnNames: b, sumc, a - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(b), max(b), count(1), count(b), compute_bit_vector_hll(b), min(sumc), max(sumc), count(sumc), compute_bit_vector_hll(sumc) keys: a (type: int) @@ -336,11 +336,11 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.4 @@ -361,20 +361,20 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col2) keys: _col0 (type: int), _col1 (type: int) - minReductionHashAggr: 0.4285714 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) Union 3 Vertex: Union 3 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_create_rewrite_agg_2.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_create_rewrite_agg_2.q.out index 4e02cb3ed46e..51ba08208bb8 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_create_rewrite_agg_2.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_create_rewrite_agg_2.q.out @@ -113,11 +113,11 @@ HiveProject(sumb=[$3], sume=[$4], a=[$0], c=[$1], d=[$2]) HiveTableScan(table=[[default, mat1]], table:alias=[default.mat1]) HiveProject(a=[$0], c=[$1], d=[$2]) HiveAggregate(group=[{0, 2, 3}]) - HiveFilter(condition=[<(1, $7.writeid)]) + HiveFilter(condition=[IS NOT TRUE(>=(1, $7.writeid))]) HiveTableScan(table=[[default, t1]], table:alias=[t1]) HiveProject(a=[$0], c=[$1], d=[$2], $f3=[$3], $f4=[$4]) HiveAggregate(group=[{0, 2, 3}], agg#0=[sum($1)], agg#1=[sum($4)]) - HiveFilter(condition=[<(1, $7.writeid)]) + HiveFilter(condition=[IS NOT TRUE(>=(1, $7.writeid))]) HiveTableScan(table=[[default, t1]], table:alias=[t1]) PREHOOK: query: EXPLAIN @@ -180,15 +180,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1 - filterExpr: (ROW__ID.writeid > 1L) (type: boolean) + filterExpr: (ROW__ID.writeid <= 1L) is not true (type: boolean) Statistics: Num rows: 11 Data size: 1044 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (ROW__ID.writeid > 1L) (type: boolean) - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (ROW__ID.writeid <= 1L) is not true (type: boolean) + Statistics: Num rows: 5 Data size: 480 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: int), d (type: string) outputColumnNames: a, c, d - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 480 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: a (type: int), c (type: int), d (type: string) minReductionHashAggr: 0.4 @@ -204,7 +204,7 @@ STAGE PLANS: Select Operator expressions: a (type: int), b (type: int), c (type: int), d (type: string), e (type: float) outputColumnNames: a, b, c, d, e - Statistics: Num rows: 3 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 520 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(b), sum(e) keys: a (type: int), c (type: int), d (type: string) diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_create_rewrite_agg_3.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_create_rewrite_agg_3.q.out index 9c3897d016e6..136129351815 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_create_rewrite_agg_3.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_create_rewrite_agg_3.q.out @@ -132,11 +132,11 @@ HiveProject(b=[$0], sumc=[$2], a=[$1]) HiveTableScan(table=[[default, mat1]], table:alias=[default.mat1]) HiveProject(b=[$1], a=[$0]) HiveAggregate(group=[{0, 1}]) - HiveFilter(condition=[<(1, $5.writeid)]) + HiveFilter(condition=[IS NOT TRUE(>=(1, $5.writeid))]) HiveTableScan(table=[[default, t1]], table:alias=[t1]) HiveProject(b=[$1], a=[$0], $f2=[$2]) HiveAggregate(group=[{0, 1}], agg#0=[sum($2)]) - HiveFilter(condition=[<(1, $5.writeid)]) + HiveFilter(condition=[IS NOT TRUE(>=(1, $5.writeid))]) HiveTableScan(table=[[default, t1]], table:alias=[t1]) PREHOOK: query: EXPLAIN @@ -198,46 +198,46 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1 - filterExpr: (ROW__ID.writeid > 1L) (type: boolean) + filterExpr: (ROW__ID.writeid <= 1L) is not true (type: boolean) properties: insertonly.fetch.bucketid TRUE Statistics: Num rows: 10 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (ROW__ID.writeid > 1L) (type: boolean) - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (ROW__ID.writeid <= 1L) is not true (type: boolean) + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), b (type: int) outputColumnNames: a, b - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: a (type: int), b (type: int) minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), b (type: int), c (type: int) outputColumnNames: a, b, c - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(c) keys: b (type: int), a (type: int) minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) @@ -256,16 +256,16 @@ STAGE PLANS: Group By Operator aggregations: sum(_col2) keys: _col0 (type: int), _col1 (type: int) - minReductionHashAggr: 0.4285714 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) Reducer 4 Execution mode: vectorized, llap @@ -275,14 +275,14 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col2 (type: bigint), _col1 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -292,7 +292,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: int) outputColumnNames: b, sumc, a - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(b), max(b), count(1), count(b), compute_bit_vector_hll(b), min(sumc), max(sumc), count(sumc), compute_bit_vector_hll(sumc) keys: a (type: int) @@ -334,11 +334,11 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.4 @@ -359,20 +359,20 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col2) keys: _col0 (type: int), _col1 (type: int) - minReductionHashAggr: 0.4285714 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) Union 3 Vertex: Union 3 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_rebuild_2.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_rebuild_2.q.out index 454c2c0af07a..4cf06e98da70 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_rebuild_2.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_rebuild_2.q.out @@ -92,10 +92,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1 - filterExpr: ((ROW__ID.writeid > 1L) and (col0 = 1)) (type: boolean) + filterExpr: ((ROW__ID.writeid <= 1L) is not true and (col0 = 1)) (type: boolean) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((ROW__ID.writeid > 1L) and (col0 = 1)) (type: boolean) + predicate: ((ROW__ID.writeid <= 1L) is not true and (col0 = 1)) (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 1 (type: int) diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_rebuild_3.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_rebuild_3.q.out index 2ef598e2d963..82b9c4fe5fce 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_rebuild_3.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_rebuild_3.q.out @@ -95,28 +95,28 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1 - filterExpr: (ROW__ID.writeid > 1L) (type: boolean) + filterExpr: (ROW__ID.writeid <= 1L) is not true (type: boolean) Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (ROW__ID.writeid > 1L) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (ROW__ID.writeid <= 1L) is not true (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: col0 (type: int), col1 (type: int) outputColumnNames: col0, col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(col1) keys: col0 (type: int) minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) @@ -132,7 +132,7 @@ STAGE PLANS: Group By Operator aggregations: sum(_col1) keys: _col0 (type: int) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE @@ -153,11 +153,11 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col1) keys: _col0 (type: int) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/llap/sketches_materialized_view_cume_dist.q.out b/ql/src/test/results/clientpositive/llap/sketches_materialized_view_cume_dist.q.out index 96f4236c95e1..c80769a4ed9d 100644 --- a/ql/src/test/results/clientpositive/llap/sketches_materialized_view_cume_dist.q.out +++ b/ql/src/test/results/clientpositive/llap/sketches_materialized_view_cume_dist.q.out @@ -544,19 +544,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: sketch_input - filterExpr: (ROW__ID.writeid > 1L) (type: boolean) + filterExpr: (ROW__ID.writeid <= 1L) is not true (type: boolean) Statistics: Num rows: 44 Data size: 3916 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (ROW__ID.writeid > 1L) (type: boolean) - Statistics: Num rows: 14 Data size: 1246 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (ROW__ID.writeid <= 1L) is not true (type: boolean) + Statistics: Num rows: 22 Data size: 1958 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: category (type: char(1)), UDFToFloat(id) (type: float) outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1246 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 22 Data size: 1958 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: ds_kll_sketch(_col1) keys: _col0 (type: char(1)) - minReductionHashAggr: 0.85714287 + minReductionHashAggr: 0.9090909 mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 458 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/llap/sketches_materialized_view_ntile.q.out b/ql/src/test/results/clientpositive/llap/sketches_materialized_view_ntile.q.out index eed5ef3f6ae3..26af84ab6b5c 100644 --- a/ql/src/test/results/clientpositive/llap/sketches_materialized_view_ntile.q.out +++ b/ql/src/test/results/clientpositive/llap/sketches_materialized_view_ntile.q.out @@ -548,19 +548,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: sketch_input - filterExpr: (ROW__ID.writeid > 1L) (type: boolean) + filterExpr: (ROW__ID.writeid <= 1L) is not true (type: boolean) Statistics: Num rows: 44 Data size: 3916 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (ROW__ID.writeid > 1L) (type: boolean) - Statistics: Num rows: 14 Data size: 1246 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (ROW__ID.writeid <= 1L) is not true (type: boolean) + Statistics: Num rows: 22 Data size: 1958 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: category (type: char(1)), UDFToFloat(id) (type: float) outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1246 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 22 Data size: 1958 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: ds_kll_sketch(_col1) keys: _col0 (type: char(1)) - minReductionHashAggr: 0.85714287 + minReductionHashAggr: 0.9090909 mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 458 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/llap/sketches_materialized_view_percentile_disc.q.out b/ql/src/test/results/clientpositive/llap/sketches_materialized_view_percentile_disc.q.out index e9452e24cfc7..91145a4bfbce 100644 --- a/ql/src/test/results/clientpositive/llap/sketches_materialized_view_percentile_disc.q.out +++ b/ql/src/test/results/clientpositive/llap/sketches_materialized_view_percentile_disc.q.out @@ -312,19 +312,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: sketch_input - filterExpr: (ROW__ID.writeid > 1L) (type: boolean) + filterExpr: (ROW__ID.writeid <= 1L) is not true (type: boolean) Statistics: Num rows: 44 Data size: 3916 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (ROW__ID.writeid > 1L) (type: boolean) - Statistics: Num rows: 14 Data size: 1246 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (ROW__ID.writeid <= 1L) is not true (type: boolean) + Statistics: Num rows: 22 Data size: 1958 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: category (type: char(1)), UDFToFloat(id) (type: float) outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1246 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 22 Data size: 1958 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: ds_kll_sketch(_col1) keys: _col0 (type: char(1)) - minReductionHashAggr: 0.85714287 + minReductionHashAggr: 0.9090909 mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 458 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/llap/sketches_materialized_view_rank.q.out b/ql/src/test/results/clientpositive/llap/sketches_materialized_view_rank.q.out index bfc3ac7dfd61..45bd756f9ec8 100644 --- a/ql/src/test/results/clientpositive/llap/sketches_materialized_view_rank.q.out +++ b/ql/src/test/results/clientpositive/llap/sketches_materialized_view_rank.q.out @@ -548,19 +548,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: sketch_input - filterExpr: (ROW__ID.writeid > 1L) (type: boolean) + filterExpr: (ROW__ID.writeid <= 1L) is not true (type: boolean) Statistics: Num rows: 44 Data size: 3916 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (ROW__ID.writeid > 1L) (type: boolean) - Statistics: Num rows: 14 Data size: 1246 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (ROW__ID.writeid <= 1L) is not true (type: boolean) + Statistics: Num rows: 22 Data size: 1958 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: category (type: char(1)), UDFToFloat(id) (type: float) outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1246 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 22 Data size: 1958 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: ds_kll_sketch(_col1) keys: _col0 (type: char(1)) - minReductionHashAggr: 0.85714287 + minReductionHashAggr: 0.9090909 mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 458 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/llap/sketches_materialized_view_rollup2.q.out b/ql/src/test/results/clientpositive/llap/sketches_materialized_view_rollup2.q.out index 96f517f59b5b..559ed289e9ec 100644 --- a/ql/src/test/results/clientpositive/llap/sketches_materialized_view_rollup2.q.out +++ b/ql/src/test/results/clientpositive/llap/sketches_materialized_view_rollup2.q.out @@ -320,19 +320,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: sketch_input - filterExpr: (ROW__ID.writeid > 1L) (type: boolean) + filterExpr: (ROW__ID.writeid <= 1L) is not true (type: boolean) Statistics: Num rows: 44 Data size: 3916 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (ROW__ID.writeid > 1L) (type: boolean) - Statistics: Num rows: 14 Data size: 1246 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (ROW__ID.writeid <= 1L) is not true (type: boolean) + Statistics: Num rows: 22 Data size: 1958 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: id (type: int), category (type: char(1)) outputColumnNames: id, category - Statistics: Num rows: 14 Data size: 1246 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 22 Data size: 1958 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: ds_hll_sketch(id), count(id) keys: category (type: char(1)) - minReductionHashAggr: 0.85714287 + minReductionHashAggr: 0.9090909 mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 2 Data size: 962 Basic stats: COMPLETE Column stats: COMPLETE From c0832dcb125afa353acc1f38cbdf7fd02d1692ee Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Tue, 1 Sep 2026 20:12:33 +0100 Subject: [PATCH 09/24] Adjust test plans (simplifications from CALCITE-7722) --- ...vectorized_dynamic_partition_pruning.q.out | 107 ++++++++---------- 1 file changed, 49 insertions(+), 58 deletions(-) diff --git a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out index 5f7f20b165dd..20ebf06aded1 100644 --- a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out @@ -1918,31 +1918,28 @@ STAGE PLANS: expressions: UDFToDouble(UDFToInteger((hr / 2.0D))) (type: double) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: double) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: double) + Select Operator + expressions: _col0 (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: double) + minReductionHashAggr: 0.4 + mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: double) - minReductionHashAggr: 0.4 - mode: hash - outputColumnNames: _col0 + Dynamic Partitioning Event Operator + Target column: hr (string) + Target Input: srcpart + Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Dynamic Partitioning Event Operator - Target column: hr (string) - Target Input: srcpart - Partition key expr: UDFToDouble(hr) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 + Target Vertex: Map 1 Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -2064,10 +2061,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (UDFToDouble(hr) * 2.0D) is not null (type: boolean) + filterExpr: UDFToDouble(hr) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (UDFToDouble(hr) * 2.0D) is not null (type: boolean) + predicate: UDFToDouble(hr) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: (UDFToDouble(hr) * 2.0D) (type: double) @@ -2285,15 +2282,12 @@ STAGE PLANS: expressions: UDFToDouble(UDFToInteger((hr / 2.0D))) (type: double) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: double) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -2415,10 +2409,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (UDFToDouble(hr) * 2.0D) is not null (type: boolean) + filterExpr: UDFToDouble(hr) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (UDFToDouble(hr) * 2.0D) is not null (type: boolean) + predicate: UDFToDouble(hr) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: (UDFToDouble(hr) * 2.0D) (type: double) @@ -2594,10 +2588,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (UDFToDouble(hr) * 2.0D) is not null (type: boolean) + filterExpr: UDFToDouble(hr) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (UDFToDouble(hr) * 2.0D) is not null (type: boolean) + predicate: UDFToDouble(hr) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: CAST( (UDFToDouble(hr) * 2.0D) AS STRING) (type: string) @@ -5918,31 +5912,28 @@ STAGE PLANS: expressions: UDFToDouble(UDFToInteger((hr / 2.0D))) (type: double) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: double) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: double) + Select Operator + expressions: _col0 (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: double) + minReductionHashAggr: 0.4 + mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: double) - minReductionHashAggr: 0.4 - mode: hash - outputColumnNames: _col0 + Dynamic Partitioning Event Operator + Target column: hr (string) + Target Input: srcpart + Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Dynamic Partitioning Event Operator - Target column: hr (string) - Target Input: srcpart - Partition key expr: UDFToDouble(hr) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 + Target Vertex: Map 1 Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -6040,10 +6031,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (UDFToDouble(hr) * 2.0D) is not null (type: boolean) + filterExpr: UDFToDouble(hr) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (UDFToDouble(hr) * 2.0D) is not null (type: boolean) + predicate: UDFToDouble(hr) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: (UDFToDouble(hr) * 2.0D) (type: double) From 320de847126448dcf060ba44cced8c05953fc7b4 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Tue, 1 Sep 2026 20:12:44 +0100 Subject: [PATCH 10/24] Adjust visitors due to CALCITE-7511 --- .../HiveRelOptMaterializationValidator.java | 15 +++++++++++---- .../translator/PlanModifierForASTConv.java | 9 +++------ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptMaterializationValidator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptMaterializationValidator.java index 29cf5488f530..ebe508d6fff1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptMaterializationValidator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptMaterializationValidator.java @@ -21,6 +21,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.SortExchange; import org.apache.calcite.rel.core.TableFunctionScan; import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.logical.LogicalAggregate; @@ -131,18 +132,24 @@ public RelNode visit(HiveAggregate aggregate) { return super.visit(aggregate); } + @Override + public RelNode visit(SortExchange sortExchange) { + if (sortExchange instanceof HiveSortExchange) { + return visit((HiveSortExchange) sortExchange); + } + return super.visit(sortExchange); + } + @Override public RelNode visit(RelNode node) { // There are several Hive RelNode types which do not have their own visit() method // defined in the HiveRelShuttle interface, which need to be handled appropriately here. - // Per jcamachorodriguez we should not encounter HiveMultiJoin/HiveSortExchange - // during these checks, so no need to add those here. + // Per jcamachorodriguez we should not encounter HiveMultiJoin + // during these checks, so no need to add it here. if (node instanceof HiveUnion) { return visit((HiveUnion) node); } else if (node instanceof HiveSortLimit) { return visit((HiveSortLimit) node); - } else if (node instanceof HiveSortExchange) { - return visit((HiveSortExchange) node); } else if (node instanceof HiveSemiJoin) { return visit((HiveSemiJoin) node); } else if (node instanceof HiveExcept) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java index 52e9b3ad2a35..78646a2a4754 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java @@ -263,12 +263,9 @@ public RelNode visit(HiveJdbcConverter conv) { } @Override - public RelNode visit(final RelNode rel) { - if (rel instanceof TableSpool spool) { - aliases.add(spool.getTable().getQualifiedName().getLast().toLowerCase()); - return spool; - } - return super.visit(rel); + public RelNode visit(TableSpool spool) { + aliases.add(spool.getTable().getQualifiedName().getLast().toLowerCase()); + return spool; } } From 0463961e08bdbb5f546553d6283ebaab28fbe281 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Wed, 2 Sep 2026 11:32:30 +0100 Subject: [PATCH 11/24] Adjust test files 'ROW__ID.writeid > 1L' ==> '(ROW__ID.writeid <= 1L) is not true' due to CALCITE-7636 --- .../hive/ql/txn/compactor/TestMaterializedViewRebuild.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMaterializedViewRebuild.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMaterializedViewRebuild.java index 5314be804b95..f8e1fba781d6 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMaterializedViewRebuild.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMaterializedViewRebuild.java @@ -56,7 +56,7 @@ public class TestMaterializedViewRebuild extends CompactorOnTezTest { private static final List INCREMENTAL_REBUILD_PLAN = Arrays.asList( "CBO PLAN:", "HiveProject(t1.a=[$0], t1.b=[$1], t1.c=[$2])", - " HiveFilter(condition=[AND(>($5.writeid, 2), OR(IS NULL($0), >($0, 0)))])", + " HiveFilter(condition=[AND(OR(IS NULL($0), >($0, 0)), IS NOT TRUE(>=(2, $5.writeid)))])", " HiveTableScan(table=[[default, t1]], table:alias=[t1])", "" ); From 47c6d947688cc5a8d01e8ee9c0bd30f23d6b41c1 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Wed, 2 Sep 2026 11:32:56 +0100 Subject: [PATCH 12/24] Adjust test file due to CALCITE-7636 --- .../results/clientpositive/perf/tpcds30tb/cte/cbo_query54.q.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ql/src/test/results/clientpositive/perf/tpcds30tb/cte/cbo_query54.q.out b/ql/src/test/results/clientpositive/perf/tpcds30tb/cte/cbo_query54.q.out index f5a2ecae98ca..32b4615371ea 100644 --- a/ql/src/test/results/clientpositive/perf/tpcds30tb/cte/cbo_query54.q.out +++ b/ql/src/test/results/clientpositive/perf/tpcds30tb/cte/cbo_query54.q.out @@ -82,7 +82,7 @@ HiveSortLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[100]) HiveProject(d_date_sk=[$0], d_month_seq=[$3]) HiveFilter(condition=[IS NOT NULL($3)]) HiveProject(d_date_sk=[$0], d_date_id=[$1], d_date=[$2], d_month_seq=[$3], d_week_seq=[$4], d_quarter_seq=[$5], d_year=[$6], d_dow=[$7], d_moy=[$8], d_dom=[$9], d_qoy=[$10], d_fy_year=[$11], d_fy_quarter_seq=[$12], d_fy_week_seq=[$13], d_day_name=[$14], d_quarter_name=[$15], d_holiday=[$16], d_weekend=[$17], d_following_holiday=[$18], d_first_dom=[$19], d_last_dom=[$20], d_same_day_ly=[$21], d_same_day_lq=[$22], d_current_day=[$23], d_current_week=[$24], d_current_month=[$25], d_current_quarter=[$26], d_current_year=[$27], BLOCK__OFFSET__INSIDE__FILE=[$28], INPUT__FILE__NAME=[$29], ROW__ID=[$30], ROW__IS__DELETED=[$31]) - HiveFilter(condition=[OR(<>(1999, $6), <>(3, $8))]) + HiveFilter(condition=[IS NOT TRUE(AND(=(1999, $6), =(3, $8)))]) HiveTableScan(table=[[default, date_dim]], table:alias=[date_dim]) HiveProject(d_date_sk=[$0], d_month_seq=[$3]) HiveTableSpool(table=[[cte, cte_suggestion_1]]) From 3b77c82befa5a52954fb0dec94962dd21665c353 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Wed, 2 Sep 2026 11:33:15 +0100 Subject: [PATCH 13/24] Adjust test plans (simplifications probably from CALCITE-7722) --- .../merge_with_null_check_on_joining_col.q.out | 2 +- .../llap/external_jdbc_table_perf.q.out | 16 ++++++++-------- .../test/results/clientpositive/llap/pcs.q.out | 12 ++++++------ .../perf/tpcds30tb/cte/cbo_query88.q.out | 18 +++++++++--------- .../perf/tpcds30tb/tez/cbo_query88.q.out | 16 ++++++++-------- .../perf/tpcds30tb/tez/query88.q.out | 4 ++-- 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_with_null_check_on_joining_col.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_with_null_check_on_joining_col.q.out index ca720205bf24..38427b55548d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_with_null_check_on_joining_col.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_with_null_check_on_joining_col.q.out @@ -155,7 +155,7 @@ HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path= HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) HiveSemiJoin(condition=[=($2, $9)], joinType=[semi]) HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6], a=[$7], b=[$8]) - HiveFilter(condition=[OR(IS NULL(OR(IS NULL($4), AND(=($4, $7), =($5, $8), >($4, 10)), AND(IS NULL($4), IS NULL($5)))), AND(OR(<>($4, $7), <>($5, $8), <=($4, 10)), IS NOT NULL($4)))]) + HiveFilter(condition=[OR(IS NULL(OR(IS NULL($4), AND(=($4, $7), =($5, $8), >($4, 10)))), AND(OR(<>($4, $7), <>($5, $8), <=($4, 10)), IS NOT NULL($4)))]) HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1], t__c=[$2]) HiveFilter(condition=[IS NOT NULL($5)]) diff --git a/ql/src/test/results/clientpositive/llap/external_jdbc_table_perf.q.out b/ql/src/test/results/clientpositive/llap/external_jdbc_table_perf.q.out index 639cb0cc6e31..2ac5b5f12f2c 100644 --- a/ql/src/test/results/clientpositive/llap/external_jdbc_table_perf.q.out +++ b/ql/src/test/results/clientpositive/llap/external_jdbc_table_perf.q.out @@ -5216,7 +5216,7 @@ WHERE "ss_hdemo_sk" IS NOT NULL AND "ss_sold_time_sk" IS NOT NULL AND "ss_store_ INNER JOIN (SELECT "hd_demo_sk" FROM (SELECT "hd_demo_sk", "hd_dep_count", "hd_vehicle_count" FROM "HOUSEHOLD_DEMOGRAPHICS") AS "t2" -WHERE "hd_vehicle_count" <= 5 AND ("hd_dep_count" = 4 AND "hd_vehicle_count" IS NOT NULL OR "hd_dep_count" = 2 AND "hd_vehicle_count" <= 2 OR "hd_dep_count" = 0 AND "hd_vehicle_count" <= 3) AND "hd_dep_count" IN (0, 2, 4) AND "hd_demo_sk" IS NOT NULL) AS "t4" ON "t1"."ss_hdemo_sk" = "t4"."hd_demo_sk" +WHERE "hd_vehicle_count" <= 5 AND ("hd_dep_count" = 4 OR "hd_dep_count" = 2 AND "hd_vehicle_count" <= 2 OR "hd_dep_count" = 0 AND "hd_vehicle_count" <= 3) AND "hd_dep_count" IN (0, 2, 4) AND "hd_demo_sk" IS NOT NULL) AS "t4" ON "t1"."ss_hdemo_sk" = "t4"."hd_demo_sk" INNER JOIN (SELECT "t_time_sk" FROM (SELECT "t_time_sk", "t_hour", "t_minute" FROM "TIME_DIM") AS "t5" @@ -5253,7 +5253,7 @@ WHERE "ss_hdemo_sk" IS NOT NULL AND "ss_sold_time_sk" IS NOT NULL AND "ss_store_ INNER JOIN (SELECT "hd_demo_sk" FROM (SELECT "hd_demo_sk", "hd_dep_count", "hd_vehicle_count" FROM "HOUSEHOLD_DEMOGRAPHICS") AS "t2" -WHERE "hd_vehicle_count" <= 5 AND ("hd_dep_count" = 4 AND "hd_vehicle_count" IS NOT NULL OR "hd_dep_count" = 2 AND "hd_vehicle_count" <= 2 OR "hd_dep_count" = 0 AND "hd_vehicle_count" <= 3) AND "hd_dep_count" IN (0, 2, 4) AND "hd_demo_sk" IS NOT NULL) AS "t4" ON "t1"."ss_hdemo_sk" = "t4"."hd_demo_sk" +WHERE "hd_vehicle_count" <= 5 AND ("hd_dep_count" = 4 OR "hd_dep_count" = 2 AND "hd_vehicle_count" <= 2 OR "hd_dep_count" = 0 AND "hd_vehicle_count" <= 3) AND "hd_dep_count" IN (0, 2, 4) AND "hd_demo_sk" IS NOT NULL) AS "t4" ON "t1"."ss_hdemo_sk" = "t4"."hd_demo_sk" INNER JOIN (SELECT "t_time_sk" FROM (SELECT "t_time_sk", "t_hour", "t_minute" FROM "TIME_DIM") AS "t5" @@ -5290,7 +5290,7 @@ WHERE "ss_hdemo_sk" IS NOT NULL AND "ss_sold_time_sk" IS NOT NULL AND "ss_store_ INNER JOIN (SELECT "hd_demo_sk" FROM (SELECT "hd_demo_sk", "hd_dep_count", "hd_vehicle_count" FROM "HOUSEHOLD_DEMOGRAPHICS") AS "t2" -WHERE "hd_vehicle_count" <= 5 AND ("hd_dep_count" = 4 AND "hd_vehicle_count" IS NOT NULL OR "hd_dep_count" = 2 AND "hd_vehicle_count" <= 2 OR "hd_dep_count" = 0 AND "hd_vehicle_count" <= 3) AND "hd_dep_count" IN (0, 2, 4) AND "hd_demo_sk" IS NOT NULL) AS "t4" ON "t1"."ss_hdemo_sk" = "t4"."hd_demo_sk" +WHERE "hd_vehicle_count" <= 5 AND ("hd_dep_count" = 4 OR "hd_dep_count" = 2 AND "hd_vehicle_count" <= 2 OR "hd_dep_count" = 0 AND "hd_vehicle_count" <= 3) AND "hd_dep_count" IN (0, 2, 4) AND "hd_demo_sk" IS NOT NULL) AS "t4" ON "t1"."ss_hdemo_sk" = "t4"."hd_demo_sk" INNER JOIN (SELECT "t_time_sk" FROM (SELECT "t_time_sk", "t_hour", "t_minute" FROM "TIME_DIM") AS "t5" @@ -5327,7 +5327,7 @@ WHERE "ss_hdemo_sk" IS NOT NULL AND "ss_sold_time_sk" IS NOT NULL AND "ss_store_ INNER JOIN (SELECT "hd_demo_sk" FROM (SELECT "hd_demo_sk", "hd_dep_count", "hd_vehicle_count" FROM "HOUSEHOLD_DEMOGRAPHICS") AS "t2" -WHERE "hd_vehicle_count" <= 5 AND ("hd_dep_count" = 4 AND "hd_vehicle_count" IS NOT NULL OR "hd_dep_count" = 2 AND "hd_vehicle_count" <= 2 OR "hd_dep_count" = 0 AND "hd_vehicle_count" <= 3) AND "hd_dep_count" IN (0, 2, 4) AND "hd_demo_sk" IS NOT NULL) AS "t4" ON "t1"."ss_hdemo_sk" = "t4"."hd_demo_sk" +WHERE "hd_vehicle_count" <= 5 AND ("hd_dep_count" = 4 OR "hd_dep_count" = 2 AND "hd_vehicle_count" <= 2 OR "hd_dep_count" = 0 AND "hd_vehicle_count" <= 3) AND "hd_dep_count" IN (0, 2, 4) AND "hd_demo_sk" IS NOT NULL) AS "t4" ON "t1"."ss_hdemo_sk" = "t4"."hd_demo_sk" INNER JOIN (SELECT "t_time_sk" FROM (SELECT "t_time_sk", "t_hour", "t_minute" FROM "TIME_DIM") AS "t5" @@ -5364,7 +5364,7 @@ WHERE "ss_hdemo_sk" IS NOT NULL AND "ss_sold_time_sk" IS NOT NULL AND "ss_store_ INNER JOIN (SELECT "hd_demo_sk" FROM (SELECT "hd_demo_sk", "hd_dep_count", "hd_vehicle_count" FROM "HOUSEHOLD_DEMOGRAPHICS") AS "t2" -WHERE "hd_vehicle_count" <= 5 AND ("hd_dep_count" = 4 AND "hd_vehicle_count" IS NOT NULL OR "hd_dep_count" = 2 AND "hd_vehicle_count" <= 2 OR "hd_dep_count" = 0 AND "hd_vehicle_count" <= 3) AND "hd_dep_count" IN (0, 2, 4) AND "hd_demo_sk" IS NOT NULL) AS "t4" ON "t1"."ss_hdemo_sk" = "t4"."hd_demo_sk" +WHERE "hd_vehicle_count" <= 5 AND ("hd_dep_count" = 4 OR "hd_dep_count" = 2 AND "hd_vehicle_count" <= 2 OR "hd_dep_count" = 0 AND "hd_vehicle_count" <= 3) AND "hd_dep_count" IN (0, 2, 4) AND "hd_demo_sk" IS NOT NULL) AS "t4" ON "t1"."ss_hdemo_sk" = "t4"."hd_demo_sk" INNER JOIN (SELECT "t_time_sk" FROM (SELECT "t_time_sk", "t_hour", "t_minute" FROM "TIME_DIM") AS "t5" @@ -5401,7 +5401,7 @@ WHERE "ss_hdemo_sk" IS NOT NULL AND "ss_sold_time_sk" IS NOT NULL AND "ss_store_ INNER JOIN (SELECT "hd_demo_sk" FROM (SELECT "hd_demo_sk", "hd_dep_count", "hd_vehicle_count" FROM "HOUSEHOLD_DEMOGRAPHICS") AS "t2" -WHERE "hd_vehicle_count" <= 5 AND ("hd_dep_count" = 4 AND "hd_vehicle_count" IS NOT NULL OR "hd_dep_count" = 2 AND "hd_vehicle_count" <= 2 OR "hd_dep_count" = 0 AND "hd_vehicle_count" <= 3) AND "hd_dep_count" IN (0, 2, 4) AND "hd_demo_sk" IS NOT NULL) AS "t4" ON "t1"."ss_hdemo_sk" = "t4"."hd_demo_sk" +WHERE "hd_vehicle_count" <= 5 AND ("hd_dep_count" = 4 OR "hd_dep_count" = 2 AND "hd_vehicle_count" <= 2 OR "hd_dep_count" = 0 AND "hd_vehicle_count" <= 3) AND "hd_dep_count" IN (0, 2, 4) AND "hd_demo_sk" IS NOT NULL) AS "t4" ON "t1"."ss_hdemo_sk" = "t4"."hd_demo_sk" INNER JOIN (SELECT "t_time_sk" FROM (SELECT "t_time_sk", "t_hour", "t_minute" FROM "TIME_DIM") AS "t5" @@ -5438,7 +5438,7 @@ WHERE "ss_hdemo_sk" IS NOT NULL AND "ss_sold_time_sk" IS NOT NULL AND "ss_store_ INNER JOIN (SELECT "hd_demo_sk" FROM (SELECT "hd_demo_sk", "hd_dep_count", "hd_vehicle_count" FROM "HOUSEHOLD_DEMOGRAPHICS") AS "t2" -WHERE "hd_vehicle_count" <= 5 AND ("hd_dep_count" = 4 AND "hd_vehicle_count" IS NOT NULL OR "hd_dep_count" = 2 AND "hd_vehicle_count" <= 2 OR "hd_dep_count" = 0 AND "hd_vehicle_count" <= 3) AND "hd_dep_count" IN (0, 2, 4) AND "hd_demo_sk" IS NOT NULL) AS "t4" ON "t1"."ss_hdemo_sk" = "t4"."hd_demo_sk" +WHERE "hd_vehicle_count" <= 5 AND ("hd_dep_count" = 4 OR "hd_dep_count" = 2 AND "hd_vehicle_count" <= 2 OR "hd_dep_count" = 0 AND "hd_vehicle_count" <= 3) AND "hd_dep_count" IN (0, 2, 4) AND "hd_demo_sk" IS NOT NULL) AS "t4" ON "t1"."ss_hdemo_sk" = "t4"."hd_demo_sk" INNER JOIN (SELECT "t_time_sk" FROM (SELECT "t_time_sk", "t_hour", "t_minute" FROM "TIME_DIM") AS "t5" @@ -5475,7 +5475,7 @@ WHERE "ss_hdemo_sk" IS NOT NULL AND "ss_sold_time_sk" IS NOT NULL AND "ss_store_ INNER JOIN (SELECT "hd_demo_sk" FROM (SELECT "hd_demo_sk", "hd_dep_count", "hd_vehicle_count" FROM "HOUSEHOLD_DEMOGRAPHICS") AS "t2" -WHERE "hd_vehicle_count" <= 5 AND ("hd_dep_count" = 4 AND "hd_vehicle_count" IS NOT NULL OR "hd_dep_count" = 2 AND "hd_vehicle_count" <= 2 OR "hd_dep_count" = 0 AND "hd_vehicle_count" <= 3) AND "hd_dep_count" IN (0, 2, 4) AND "hd_demo_sk" IS NOT NULL) AS "t4" ON "t1"."ss_hdemo_sk" = "t4"."hd_demo_sk" +WHERE "hd_vehicle_count" <= 5 AND ("hd_dep_count" = 4 OR "hd_dep_count" = 2 AND "hd_vehicle_count" <= 2 OR "hd_dep_count" = 0 AND "hd_vehicle_count" <= 3) AND "hd_dep_count" IN (0, 2, 4) AND "hd_demo_sk" IS NOT NULL) AS "t4" ON "t1"."ss_hdemo_sk" = "t4"."hd_demo_sk" INNER JOIN (SELECT "t_time_sk" FROM (SELECT "t_time_sk", "t_hour", "t_minute" FROM "TIME_DIM") AS "t5" diff --git a/ql/src/test/results/clientpositive/llap/pcs.q.out b/ql/src/test/results/clientpositive/llap/pcs.q.out index 14b5bc3e1f12..b0beab664567 100644 --- a/ql/src/test/results/clientpositive/llap/pcs.q.out +++ b/ql/src/test/results/clientpositive/llap/pcs.q.out @@ -1407,7 +1407,7 @@ POSTHOOK: Input: default@pcs_t1@ds=2000-04-10 #### A masked pattern was here #### OPTIMIZED SQL: SELECT `ds` FROM `default`.`pcs_t1` -WHERE `ds` <> '2000-04-08' AND `key` = 3 OR (`ds` = '2000-04-08' OR `key` IS NOT NULL) AND `key` = 2 +WHERE `key` = 2 OR `ds` <> '2000-04-08' AND `key` = 3 STAGE DEPENDENCIES: Stage-0 is a root stage @@ -1521,11 +1521,11 @@ STAGE PLANS: Processor Tree: TableScan alias: pcs_t1 - filterExpr: (((ds <> '2000-04-08') and (key = 3)) or (((ds = '2000-04-08') or key is not null) and (key = 2))) (type: boolean) + filterExpr: ((key = 2) or ((ds <> '2000-04-08') and (key = 3))) (type: boolean) GatherStats: false Filter Operator isSamplingPred: false - predicate: (((ds <> '2000-04-08') and (key = 3)) or (((ds = '2000-04-08') or key is not null) and (key = 2))) (type: boolean) + predicate: ((key = 2) or ((ds <> '2000-04-08') and (key = 3))) (type: boolean) Select Operator expressions: ds (type: string) outputColumnNames: _col0 @@ -1564,7 +1564,7 @@ POSTHOOK: Input: default@pcs_t1@ds=2000-04-10 #### A masked pattern was here #### OPTIMIZED SQL: SELECT `ds` FROM `default`.`pcs_t1` -WHERE `key` = 3 OR ((`ds` = '2000-04-08' OR `key` IS NOT NULL) AND `key` = 2 OR `ds` <> '2000-04-08' AND `key` = 3) AND `key` + 5 > 0 +WHERE `key` = 3 OR (`key` = 2 OR `ds` <> '2000-04-08' AND `key` = 3) AND `key` + 5 > 0 STAGE DEPENDENCIES: Stage-0 is a root stage @@ -1678,11 +1678,11 @@ STAGE PLANS: Processor Tree: TableScan alias: pcs_t1 - filterExpr: ((key = 3) or (((((ds = '2000-04-08') or key is not null) and (key = 2)) or ((ds <> '2000-04-08') and (key = 3))) and ((key + 5) > 0))) (type: boolean) + filterExpr: ((key = 3) or (((key = 2) or ((ds <> '2000-04-08') and (key = 3))) and ((key + 5) > 0))) (type: boolean) GatherStats: false Filter Operator isSamplingPred: false - predicate: ((key = 3) or (((((ds = '2000-04-08') or key is not null) and (key = 2)) or ((ds <> '2000-04-08') and (key = 3))) and ((key + 5) > 0))) (type: boolean) + predicate: ((key = 3) or (((key = 2) or ((ds <> '2000-04-08') and (key = 3))) and ((key + 5) > 0))) (type: boolean) Select Operator expressions: ds (type: string) outputColumnNames: _col0 diff --git a/ql/src/test/results/clientpositive/perf/tpcds30tb/cte/cbo_query88.q.out b/ql/src/test/results/clientpositive/perf/tpcds30tb/cte/cbo_query88.q.out index bd8506122233..6e928655a5de 100644 --- a/ql/src/test/results/clientpositive/perf/tpcds30tb/cte/cbo_query88.q.out +++ b/ql/src/test/results/clientpositive/perf/tpcds30tb/cte/cbo_query88.q.out @@ -1,6 +1,6 @@ CTE Suggestion: HiveProject(hd_demo_sk=[$0]) - HiveFilter(condition=[AND(IN($3, 0, 1, 3), <=($4, 5), OR(AND(=($3, 3), IS NOT NULL($4)), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) + HiveFilter(condition=[AND(IN($3, 0, 1, 3), <=($4, 5), OR(=($3, 3), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) HiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) CTE Suggestion: @@ -42,7 +42,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ HiveFilter(condition=[AND(=($3, 8), >=($4, 30))]) HiveTableScan(table=[[default, time_dim]], table:alias=[time_dim]) HiveProject(hd_demo_sk=[$0]) - HiveFilter(condition=[AND(<=($4, 5), OR(=($3, 0), =($3, 1), =($3, 3)), OR(AND(=($3, 3), IS NOT NULL($4)), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) + HiveFilter(condition=[AND(<=($4, 5), OR(=($3, 0), =($3, 1), =($3, 3)), OR(=($3, 3), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) HiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) HiveTableSpool(table=[[cte, cte_suggestion_1]]) HiveProject(s_store_sk=[$0]) @@ -58,7 +58,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ HiveFilter(condition=[AND(=($3, 12), <($4, 30))]) HiveTableScan(table=[[default, time_dim]], table:alias=[time_dim]) HiveProject(hd_demo_sk=[$0]) - HiveFilter(condition=[AND(<=($4, 5), OR(=($3, 0), =($3, 1), =($3, 3)), OR(AND(=($3, 3), IS NOT NULL($4)), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) + HiveFilter(condition=[AND(<=($4, 5), OR(=($3, 0), =($3, 1), =($3, 3)), OR(=($3, 3), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) HiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) HiveTableScan(table=[[cte, cte_suggestion_1]], table:alias=[cte_suggestion_1]) HiveProject($f0=[$0]) @@ -71,7 +71,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ HiveFilter(condition=[AND(=($3, 11), >=($4, 30))]) HiveTableScan(table=[[default, time_dim]], table:alias=[time_dim]) HiveProject(hd_demo_sk=[$0]) - HiveFilter(condition=[AND(<=($4, 5), OR(=($3, 0), =($3, 1), =($3, 3)), OR(AND(=($3, 3), IS NOT NULL($4)), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) + HiveFilter(condition=[AND(<=($4, 5), OR(=($3, 0), =($3, 1), =($3, 3)), OR(=($3, 3), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) HiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) HiveTableScan(table=[[cte, cte_suggestion_1]], table:alias=[cte_suggestion_1]) HiveProject($f0=[$0]) @@ -84,7 +84,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ HiveFilter(condition=[AND(=($3, 11), <($4, 30))]) HiveTableScan(table=[[default, time_dim]], table:alias=[time_dim]) HiveProject(hd_demo_sk=[$0]) - HiveFilter(condition=[AND(<=($4, 5), OR(=($3, 0), =($3, 1), =($3, 3)), OR(AND(=($3, 3), IS NOT NULL($4)), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) + HiveFilter(condition=[AND(<=($4, 5), OR(=($3, 0), =($3, 1), =($3, 3)), OR(=($3, 3), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) HiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) HiveTableScan(table=[[cte, cte_suggestion_1]], table:alias=[cte_suggestion_1]) HiveProject($f0=[$0]) @@ -97,7 +97,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ HiveFilter(condition=[AND(=($3, 10), >=($4, 30))]) HiveTableScan(table=[[default, time_dim]], table:alias=[time_dim]) HiveProject(hd_demo_sk=[$0]) - HiveFilter(condition=[AND(<=($4, 5), OR(=($3, 0), =($3, 1), =($3, 3)), OR(AND(=($3, 3), IS NOT NULL($4)), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) + HiveFilter(condition=[AND(<=($4, 5), OR(=($3, 0), =($3, 1), =($3, 3)), OR(=($3, 3), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) HiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) HiveTableScan(table=[[cte, cte_suggestion_1]], table:alias=[cte_suggestion_1]) HiveProject($f0=[$0]) @@ -110,7 +110,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ HiveFilter(condition=[AND(=($3, 10), <($4, 30))]) HiveTableScan(table=[[default, time_dim]], table:alias=[time_dim]) HiveProject(hd_demo_sk=[$0]) - HiveFilter(condition=[AND(<=($4, 5), OR(=($3, 0), =($3, 1), =($3, 3)), OR(AND(=($3, 3), IS NOT NULL($4)), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) + HiveFilter(condition=[AND(<=($4, 5), OR(=($3, 0), =($3, 1), =($3, 3)), OR(=($3, 3), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) HiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) HiveTableScan(table=[[cte, cte_suggestion_1]], table:alias=[cte_suggestion_1]) HiveProject($f0=[$0]) @@ -123,7 +123,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ HiveFilter(condition=[AND(=($3, 9), >=($4, 30))]) HiveTableScan(table=[[default, time_dim]], table:alias=[time_dim]) HiveProject(hd_demo_sk=[$0]) - HiveFilter(condition=[AND(<=($4, 5), OR(=($3, 0), =($3, 1), =($3, 3)), OR(AND(=($3, 3), IS NOT NULL($4)), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) + HiveFilter(condition=[AND(<=($4, 5), OR(=($3, 0), =($3, 1), =($3, 3)), OR(=($3, 3), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) HiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) HiveTableScan(table=[[cte, cte_suggestion_1]], table:alias=[cte_suggestion_1]) HiveProject($f0=[$0]) @@ -136,7 +136,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ HiveFilter(condition=[AND(=($3, 9), <($4, 30))]) HiveTableScan(table=[[default, time_dim]], table:alias=[time_dim]) HiveProject(hd_demo_sk=[$0]) - HiveFilter(condition=[AND(<=($4, 5), OR(=($3, 0), =($3, 1), =($3, 3)), OR(AND(=($3, 3), IS NOT NULL($4)), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) + HiveFilter(condition=[AND(<=($4, 5), OR(=($3, 0), =($3, 1), =($3, 3)), OR(=($3, 3), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) HiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) HiveTableScan(table=[[cte, cte_suggestion_1]], table:alias=[cte_suggestion_1]) diff --git a/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/cbo_query88.q.out b/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/cbo_query88.q.out index d925d7e9efca..d65982ba469e 100644 --- a/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/cbo_query88.q.out +++ b/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/cbo_query88.q.out @@ -26,7 +26,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ HiveFilter(condition=[AND(=($3, 8), >=($4, 30))]) HiveTableScan(table=[[default, time_dim]], table:alias=[time_dim]) HiveProject(hd_demo_sk=[$0]) - HiveFilter(condition=[AND(<=($4, 5), IN($3, 0, 1, 3), OR(AND(=($3, 3), IS NOT NULL($4)), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) + HiveFilter(condition=[AND(<=($4, 5), IN($3, 0, 1, 3), OR(=($3, 3), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) HiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) HiveProject(s_store_sk=[$0]) HiveFilter(condition=[=($5, 'ese')]) @@ -43,7 +43,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ HiveFilter(condition=[AND(=($3, 12), <($4, 30))]) HiveTableScan(table=[[default, time_dim]], table:alias=[time_dim]) HiveProject(hd_demo_sk=[$0]) - HiveFilter(condition=[AND(<=($4, 5), IN($3, 0, 1, 3), OR(AND(=($3, 3), IS NOT NULL($4)), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) + HiveFilter(condition=[AND(<=($4, 5), IN($3, 0, 1, 3), OR(=($3, 3), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) HiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) HiveProject(s_store_sk=[$0]) HiveFilter(condition=[=($5, 'ese')]) @@ -60,7 +60,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ HiveFilter(condition=[AND(=($3, 11), >=($4, 30))]) HiveTableScan(table=[[default, time_dim]], table:alias=[time_dim]) HiveProject(hd_demo_sk=[$0]) - HiveFilter(condition=[AND(<=($4, 5), IN($3, 0, 1, 3), OR(AND(=($3, 3), IS NOT NULL($4)), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) + HiveFilter(condition=[AND(<=($4, 5), IN($3, 0, 1, 3), OR(=($3, 3), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) HiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) HiveProject(s_store_sk=[$0]) HiveFilter(condition=[=($5, 'ese')]) @@ -77,7 +77,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ HiveFilter(condition=[AND(=($3, 11), <($4, 30))]) HiveTableScan(table=[[default, time_dim]], table:alias=[time_dim]) HiveProject(hd_demo_sk=[$0]) - HiveFilter(condition=[AND(<=($4, 5), IN($3, 0, 1, 3), OR(AND(=($3, 3), IS NOT NULL($4)), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) + HiveFilter(condition=[AND(<=($4, 5), IN($3, 0, 1, 3), OR(=($3, 3), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) HiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) HiveProject(s_store_sk=[$0]) HiveFilter(condition=[=($5, 'ese')]) @@ -94,7 +94,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ HiveFilter(condition=[AND(=($3, 10), >=($4, 30))]) HiveTableScan(table=[[default, time_dim]], table:alias=[time_dim]) HiveProject(hd_demo_sk=[$0]) - HiveFilter(condition=[AND(<=($4, 5), IN($3, 0, 1, 3), OR(AND(=($3, 3), IS NOT NULL($4)), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) + HiveFilter(condition=[AND(<=($4, 5), IN($3, 0, 1, 3), OR(=($3, 3), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) HiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) HiveProject(s_store_sk=[$0]) HiveFilter(condition=[=($5, 'ese')]) @@ -111,7 +111,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ HiveFilter(condition=[AND(=($3, 10), <($4, 30))]) HiveTableScan(table=[[default, time_dim]], table:alias=[time_dim]) HiveProject(hd_demo_sk=[$0]) - HiveFilter(condition=[AND(<=($4, 5), IN($3, 0, 1, 3), OR(AND(=($3, 3), IS NOT NULL($4)), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) + HiveFilter(condition=[AND(<=($4, 5), IN($3, 0, 1, 3), OR(=($3, 3), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) HiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) HiveProject(s_store_sk=[$0]) HiveFilter(condition=[=($5, 'ese')]) @@ -128,7 +128,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ HiveFilter(condition=[AND(=($3, 9), >=($4, 30))]) HiveTableScan(table=[[default, time_dim]], table:alias=[time_dim]) HiveProject(hd_demo_sk=[$0]) - HiveFilter(condition=[AND(<=($4, 5), IN($3, 0, 1, 3), OR(AND(=($3, 3), IS NOT NULL($4)), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) + HiveFilter(condition=[AND(<=($4, 5), IN($3, 0, 1, 3), OR(=($3, 3), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) HiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) HiveProject(s_store_sk=[$0]) HiveFilter(condition=[=($5, 'ese')]) @@ -145,7 +145,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ HiveFilter(condition=[AND(=($3, 9), <($4, 30))]) HiveTableScan(table=[[default, time_dim]], table:alias=[time_dim]) HiveProject(hd_demo_sk=[$0]) - HiveFilter(condition=[AND(<=($4, 5), IN($3, 0, 1, 3), OR(AND(=($3, 3), IS NOT NULL($4)), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) + HiveFilter(condition=[AND(<=($4, 5), IN($3, 0, 1, 3), OR(=($3, 3), AND(=($3, 0), <=($4, 2)), AND(=($3, 1), <=($4, 3))))]) HiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) HiveProject(s_store_sk=[$0]) HiveFilter(condition=[=($5, 'ese')]) diff --git a/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/query88.q.out b/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/query88.q.out index f9ab32981493..d0cfbe031dbf 100644 --- a/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/query88.q.out +++ b/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/query88.q.out @@ -497,10 +497,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: household_demographics - filterExpr: ((hd_vehicle_count <= 5) and (hd_dep_count) IN (0, 1, 3) and (((hd_dep_count = 3) and hd_vehicle_count is not null) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3)))) (type: boolean) + filterExpr: ((hd_vehicle_count <= 5) and (hd_dep_count) IN (0, 1, 3) and ((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3)))) (type: boolean) Statistics: Num rows: 7200 Data size: 115200 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((hd_vehicle_count <= 5) and (hd_dep_count) IN (0, 1, 3) and (((hd_dep_count = 3) and hd_vehicle_count is not null) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3)))) (type: boolean) + predicate: ((hd_vehicle_count <= 5) and (hd_dep_count) IN (0, 1, 3) and ((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3)))) (type: boolean) Statistics: Num rows: 1728 Data size: 27648 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: hd_demo_sk (type: bigint) From 849d488fb115a146937fdc46d5e2e5efc1a885a8 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Wed, 2 Sep 2026 18:17:46 +0100 Subject: [PATCH 14/24] Adjustments due to CALCITE-7636 --- .../views/HiveRowIsDeletedPropagator.java | 36 ++++++++++++++++--- .../hive/ql/plan/ExprNodeDescUtils.java | 19 ++++++++-- .../materialized_view_create_rewrite_6.q.out | 12 +++---- ...ed_view_create_rewrite_6_aggr_2joins.q.out | 12 +++---- ...ed_view_create_rewrite_6_aggr_3joins.q.out | 26 +++++++------- .../materialized_view_create_rewrite_9.q.out | 12 +++---- 6 files changed, 79 insertions(+), 38 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveRowIsDeletedPropagator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveRowIsDeletedPropagator.java index cf6ce68e1d6f..4e028729fcdd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveRowIsDeletedPropagator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveRowIsDeletedPropagator.java @@ -177,12 +177,21 @@ public RelNode visit(HiveProject project, Context context) { // HiveJoin(condition=[=($0, $8)], joinType=[inner], algorithm=[none], cost=[not available]) // Check the filter condition and collect operands of OR expressions referencing only one column public RelNode visit(HiveFilter filter, Context context) { - RexNode condition = filter.getCondition(); + // CALCITE-7636 changed the compensation predicate produced by the MV rewrite from + // OR(<(N, $t1.writeid), <(N, $t2.writeid)) + // (i.e. the DeMorgan-expanded form of NOT(AND(<=($t1.writeid, N), <=($t2.writeid, N)))) + // to the nullable-aware semantically equivalent + // IS_NOT_TRUE(AND(<=($t1.writeid, N), <=($t2.writeid, N))). + // The logic below extracts per-column predicates either directly (single table changed) + // or from the operands of a top-level OR (multiple tables changed). Normalize the new + // IS_NOT_TRUE shape back into that OR-of-per-column-predicates shape so downstream code + // is agnostic to which Calcite version produced the plan. + RexNode condition = normalizeCompensationPredicate(filter.getCondition()); // The condition might be a single predicate on the rowId (if only one table changed) - RexInputRef rexInputRef = findPossibleRowIdRef(filter.getCondition()); + RexInputRef rexInputRef = findPossibleRowIdRef(condition); if (rexInputRef != null) { - context.rowIdPredicates.put(rexInputRef.getIndex(), filter.getCondition()); + context.rowIdPredicates.put(rexInputRef.getIndex(), condition); return visitChild(filter, 0, filter.getInput(0), context); } @@ -190,7 +199,7 @@ public RelNode visit(HiveFilter filter, Context context) { return visitChild(filter, 0, filter.getInput(0), context); } - for (RexNode operand : ((RexCall)condition).operands) { + for (RexNode operand : ((RexCall) condition).operands) { RexInputRef inputRef = findPossibleRowIdRef(operand); if (inputRef != null) { context.rowIdPredicates.put(inputRef.getIndex(), operand); @@ -200,6 +209,25 @@ public RelNode visit(HiveFilter filter, Context context) { return visitChild(filter, 0, filter.getInput(0), context); } + // Convert IS_NOT_TRUE(AND(a, b, ...)) into OR(IS_NOT_TRUE(a), IS_NOT_TRUE(b), ...) and leave. + // This DeMorgan-style rewrite is null-safe under three-valued logic. + private RexNode normalizeCompensationPredicate(RexNode condition) { + if (!condition.isA(SqlKind.IS_NOT_TRUE)) { + return condition; + } + RexBuilder rexBuilder = relBuilder.getRexBuilder(); + RexNode inner = ((RexCall) condition).operands.get(0); + if (!inner.isA(SqlKind.AND)) { + // Single-column IS_NOT_TRUE(x) — leave as is; findPossibleRowIdRef still recognizes it. + return condition; + } + List disjuncts = new ArrayList<>(((RexCall) inner).operands.size()); + for (RexNode op : ((RexCall) inner).operands) { + disjuncts.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_TRUE, op)); + } + return rexBuilder.makeCall(SqlStdOperatorTable.OR, disjuncts); + } + private RexInputRef findPossibleRowIdRef(RexNode operand) { Set inputRefs = findRexInputRefs(operand); if (inputRefs.size() != 1) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java index b06c35c8e7bf..1173d790fc61 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java @@ -225,7 +225,8 @@ public static boolean containsPredicate(ExprNodeDesc source, ExprNodeDesc predic * The method is equivalent to calling: {@code and(Arrays.asList(e1, e2))} *

*/ - public static ExprNodeGenericFuncDesc and(ExprNodeDesc e1, ExprNodeDesc e2) { + // TODO rql + public static ExprNodeGenericFuncDesc /* ExprNodeDesc */ and(ExprNodeDesc e1, ExprNodeDesc e2) { return and(Arrays.asList(e1, e2)); } @@ -237,11 +238,18 @@ public static ExprNodeGenericFuncDesc and(ExprNodeDesc e1, ExprNodeDesc e2) { * * TODO: Replace mergePredicates ? */ - public static ExprNodeGenericFuncDesc and(List exps) { + // TODO rql + public static ExprNodeGenericFuncDesc /* ExprNodeDesc */ and(List exps) { List flatExps = new ArrayList<>(); for (ExprNodeDesc e : exps) { split(e, flatExps); } + // Flattening/deduplication may collapse the operand list; AND/OR require at least + // two operands (GenericUDFOPAnd#initialize enforces this), so unwrap trivial cases. + // TODO rql + /* if (flatExps.size() == 1) { + return flatExps.get(0); + } */ return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, new GenericUDFOPAnd(), "and", flatExps); } @@ -252,11 +260,16 @@ public static ExprNodeGenericFuncDesc and(List exps) { * Output: OR(AND(A, B), C, D, E, F) * */ - public static ExprNodeGenericFuncDesc or(List exps) { + public static ExprNodeDesc or(List exps) { List flatExps = new ArrayList<>(); for (ExprNodeDesc e : exps) { split(e, flatExps, FunctionRegistry::isOpOr); } + // Flattening/deduplication may collapse the operand list; AND/OR require at least + // two operands (GenericUDFOPOr#initialize enforces this), so unwrap trivial cases. + if (flatExps.size() == 1) { + return flatExps.get(0); + } return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, new GenericUDFOPOr(), "or", flatExps); } diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_6.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_6.q.out index c2bd14a8df62..4a7134e17a44 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_6.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_6.q.out @@ -255,11 +255,11 @@ HiveProject(t1.a=[$5], _c1=[CAST(CASE(IS NULL($1), $6, IS NULL($6), $1, +($6, $1 HiveProject(a=[$0], $f1=[$1], $f2=[$2], $f3=[$3]) HiveAggregate(group=[{0}], agg#0=[SUM($1)], agg#1=[SUM($2)], agg#2=[SUM($3)]) HiveProject(a=[$0], $f4=[CASE(OR($3, $7), *(-1, $1), $1)], $f5=[CASE(OR($3, $7), *(-1, CASE(IS NULL($1), 0, 1)), CASE(IS NULL($1), 0, 1))], $f6=[CASE(OR($3, $7), -1, 1)]) - HiveJoin(condition=[AND(=($0, $5), OR(AND(NOT($3), NOT($7)), AND(NOT($4), NOT($8))), OR(<(3, $2.writeid), <(3, $6.writeid)))], joinType=[inner], algorithm=[none], cost=[not available]) - HiveProject(a=[$0], b=[$1], ROW__ID=[$5], _deleted=[AND($6, <(3, $5.writeid))], _inserted=[AND(<(3, $5.writeid), NOT($6))]) + HiveJoin(condition=[AND(=($0, $5), OR(AND(NOT($3), NOT($7)), AND(NOT($4), NOT($8))), IS NOT TRUE(AND(>=(3, $2.writeid), >=(3, $6.writeid))))], joinType=[inner], algorithm=[none], cost=[not available]) + HiveProject(a=[$0], b=[$1], ROW__ID=[$5], _deleted=[AND($6, IS NOT TRUE(>=(3, $5.writeid)))], _inserted=[AND(IS NOT TRUE(>=(3, $5.writeid)), NOT($6))]) HiveFilter(condition=[IS NOT NULL($0)]) HiveTableScan(table=[[default, t1]], table:alias=[t1]) - HiveProject(a=[$0], ROW__ID=[$4], _deleted=[AND($5, <(3, $4.writeid))], _inserted=[AND(<(3, $4.writeid), NOT($5))]) + HiveProject(a=[$0], ROW__ID=[$4], _deleted=[AND($5, IS NOT TRUE(>=(3, $4.writeid)))], _inserted=[AND(IS NOT TRUE(>=(3, $4.writeid)), NOT($5))]) HiveFilter(condition=[IS NOT NULL($0)]) HiveTableScan(table=[[default, t2]], table:alias=[t2]) @@ -336,7 +336,7 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 9 Data size: 837 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: char(15)), ROW__ID (type: struct), (ROW__IS__DELETED and (ROW__ID.writeid > 3L)) (type: boolean), ((ROW__ID.writeid > 3L) and (not ROW__IS__DELETED)) (type: boolean) + expressions: a (type: char(15)), ROW__ID (type: struct), (ROW__IS__DELETED and (ROW__ID.writeid <= 3L) is not true) (type: boolean), ((ROW__ID.writeid <= 3L) is not true and (not ROW__IS__DELETED)) (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 9 Data size: 1593 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator @@ -360,7 +360,7 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 7 Data size: 1211 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: char(15)), b (type: decimal(7,2)), ROW__ID (type: struct), (ROW__IS__DELETED and (ROW__ID.writeid > 3L)) (type: boolean), ((ROW__ID.writeid > 3L) and (not ROW__IS__DELETED)) (type: boolean) + expressions: a (type: char(15)), b (type: decimal(7,2)), ROW__ID (type: struct), (ROW__IS__DELETED and (ROW__ID.writeid <= 3L) is not true) (type: boolean), ((ROW__ID.writeid <= 3L) is not true and (not ROW__IS__DELETED)) (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 7 Data size: 1799 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator @@ -552,7 +552,7 @@ STAGE PLANS: 0 _col0 (type: char(15)) 1 _col0 (type: char(15)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col6, _col7, _col8 - residual filter predicates: {(((not _col3) and (not _col7)) or ((not _col4) and (not _col8)))} {((_col2.writeid > 3L) or (_col6.writeid > 3L))} + residual filter predicates: {(((not _col3) and (not _col7)) or ((not _col4) and (not _col8)))} {((_col2.writeid <= 3L) and (_col6.writeid <= 3L)) is not true} Statistics: Num rows: 1 Data size: 373 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: char(15)), if((_col3 or _col7), (-1 * _col1), _col1) (type: decimal(17,2)), if((_col3 or _col7), (-1 * if(_col1 is null, 0, 1)), if(_col1 is null, 0, 1)) (type: int), if((_col3 or _col7), -1, 1) (type: int) diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_6_aggr_2joins.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_6_aggr_2joins.q.out index e499f726ac04..0d3c9b2cb99e 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_6_aggr_2joins.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_6_aggr_2joins.q.out @@ -136,11 +136,11 @@ HiveProject(cmv_basetable_n6.a=[$5], cmv_basetable_2_n3.c=[$6], cmv_basetable_2_ HiveProject(a=[$0], c0=[$1], b0=[$2], $f3=[$3]) HiveAggregate(group=[{0, 1, 2}], agg#0=[SUM($3)]) HiveProject(a=[$0], c0=[$6], b0=[$5], $f5=[CASE(OR($2, $8), -1, 1)]) - HiveJoin(condition=[AND(=($0, $4), OR(AND(NOT($2), NOT($8)), AND(NOT($3), NOT($9))), OR(<(1, $7.writeid), <(1, $1.writeid)))], joinType=[inner], algorithm=[none], cost=[not available]) - HiveProject(a=[$0], ROW__ID=[$6], _deleted=[AND($7, <(1, $6.writeid))], _inserted=[AND(<(1, $6.writeid), NOT($7))]) + HiveJoin(condition=[AND(=($0, $4), OR(AND(NOT($2), NOT($8)), AND(NOT($3), NOT($9))), IS NOT TRUE(AND(>=(1, $7.writeid), >=(1, $1.writeid))))], joinType=[inner], algorithm=[none], cost=[not available]) + HiveProject(a=[$0], ROW__ID=[$6], _deleted=[AND($7, IS NOT TRUE(>=(1, $6.writeid)))], _inserted=[AND(IS NOT TRUE(>=(1, $6.writeid)), NOT($7))]) HiveFilter(condition=[IS NOT NULL($0)]) HiveTableScan(table=[[default, cmv_basetable_n6]], table:alias=[cmv_basetable_n6]) - HiveProject(a=[$0], b=[$1], c=[$2], ROW__ID=[$6], _deleted=[AND($7, <(1, $6.writeid))], _inserted=[AND(<(1, $6.writeid), NOT($7))]) + HiveProject(a=[$0], b=[$1], c=[$2], ROW__ID=[$6], _deleted=[AND($7, IS NOT TRUE(>=(1, $6.writeid)))], _inserted=[AND(IS NOT TRUE(>=(1, $6.writeid)), NOT($7))]) HiveFilter(condition=[IS NOT NULL($0)]) HiveTableScan(table=[[default, cmv_basetable_2_n3]], table:alias=[cmv_basetable_2_n3]) @@ -217,7 +217,7 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 2 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: int), b (type: varchar(256)), c (type: decimal(10,2)), ROW__ID (type: struct), (ROW__IS__DELETED and (ROW__ID.writeid > 1L)) (type: boolean), ((ROW__ID.writeid > 1L) and (not ROW__IS__DELETED)) (type: boolean) + expressions: a (type: int), b (type: varchar(256)), c (type: decimal(10,2)), ROW__ID (type: struct), (ROW__IS__DELETED and (ROW__ID.writeid <= 1L) is not true) (type: boolean), ((ROW__ID.writeid <= 1L) is not true and (not ROW__IS__DELETED)) (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 576 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator @@ -241,7 +241,7 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: int), ROW__ID (type: struct), (ROW__IS__DELETED and (ROW__ID.writeid > 1L)) (type: boolean), ((ROW__ID.writeid > 1L) and (not ROW__IS__DELETED)) (type: boolean) + expressions: a (type: int), ROW__ID (type: struct), (ROW__IS__DELETED and (ROW__ID.writeid <= 1L) is not true) (type: boolean), ((ROW__ID.writeid <= 1L) is not true and (not ROW__IS__DELETED)) (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 4 Data size: 352 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator @@ -433,7 +433,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col5, _col6, _col7, _col8, _col9 - residual filter predicates: {(((not _col2) and (not _col8)) or ((not _col3) and (not _col9)))} {((_col7.writeid > 1L) or (_col1.writeid > 1L))} + residual filter predicates: {(((not _col2) and (not _col8)) or ((not _col3) and (not _col9)))} {((_col7.writeid <= 1L) and (_col1.writeid <= 1L)) is not true} Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col6 (type: decimal(10,2)), _col5 (type: varchar(256)), if((_col2 or _col8), -1, 1) (type: int) diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_6_aggr_3joins.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_6_aggr_3joins.q.out index f1678e3c62a3..908cfeeaddcd 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_6_aggr_3joins.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_6_aggr_3joins.q.out @@ -166,15 +166,15 @@ HiveProject(cmv_basetable_n6.a=[$4], cmv_basetable_2_n3.c=[$5], _c2=[CASE(IS NUL HiveProject(a=[$0], c0=[$1], $f2=[$2]) HiveAggregate(group=[{0, 1}], agg#0=[SUM($2)]) HiveProject(a=[$9], c0=[$5], $f4=[CASE(OR($12, $7, $2), -1, 1)]) - HiveJoin(condition=[AND(=($9, $4), OR(>($10, 10:DECIMAL(2, 0)), >($5, 10:DECIMAL(2, 0))), OR(AND(NOT($12), NOT($7)), AND(NOT($13), NOT($8))), OR(AND(NOT($12), NOT($7), NOT($2)), AND(NOT($13), NOT($8), NOT($3))), OR(<(1, $6.writeid), <(1, $11.writeid), <(1, $1.writeid)))], joinType=[inner], algorithm=[none], cost=[not available]) + HiveJoin(condition=[AND(=($9, $4), OR(>($10, 10:DECIMAL(2, 0)), >($5, 10:DECIMAL(2, 0))), OR(AND(NOT($12), NOT($7)), AND(NOT($13), NOT($8))), OR(AND(NOT($12), NOT($7), NOT($2)), AND(NOT($13), NOT($8), NOT($3))), IS NOT TRUE(AND(>=(1, $6.writeid), >=(1, $11.writeid), >=(1, $1.writeid))))], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($0, $4)], joinType=[inner], algorithm=[none], cost=[not available]) - HiveProject(a=[$0], ROW__ID=[$5], _deleted=[AND($6, <(1, $5.writeid))], _inserted=[AND(<(1, $5.writeid), NOT($6))]) + HiveProject(a=[$0], ROW__ID=[$5], _deleted=[AND($6, IS NOT TRUE(>=(1, $5.writeid)))], _inserted=[AND(IS NOT TRUE(>=(1, $5.writeid)), NOT($6))]) HiveFilter(condition=[IS NOT NULL($0)]) HiveTableScan(table=[[default, t3]], table:alias=[t3]) - HiveProject(a=[$0], c=[$2], ROW__ID=[$6], _deleted=[AND($7, <(1, $6.writeid))], _inserted=[AND(<(1, $6.writeid), NOT($7))]) + HiveProject(a=[$0], c=[$2], ROW__ID=[$6], _deleted=[AND($7, IS NOT TRUE(>=(1, $6.writeid)))], _inserted=[AND(IS NOT TRUE(>=(1, $6.writeid)), NOT($7))]) HiveFilter(condition=[IS NOT NULL($0)]) HiveTableScan(table=[[default, cmv_basetable_2_n3]], table:alias=[cmv_basetable_2_n3]) - HiveProject(a=[$0], c=[$2], ROW__ID=[$6], _deleted=[AND($7, <(1, $6.writeid))], _inserted=[AND(<(1, $6.writeid), NOT($7))]) + HiveProject(a=[$0], c=[$2], ROW__ID=[$6], _deleted=[AND($7, IS NOT TRUE(>=(1, $6.writeid)))], _inserted=[AND(IS NOT TRUE(>=(1, $6.writeid)), NOT($7))]) HiveFilter(condition=[IS NOT NULL($0)]) HiveTableScan(table=[[default, cmv_basetable_n6]], table:alias=[cmv_basetable_n6]) @@ -254,7 +254,7 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: int), c (type: decimal(10,2)), ROW__ID (type: struct), (ROW__IS__DELETED and (ROW__ID.writeid > 1L)) (type: boolean), ((ROW__ID.writeid > 1L) and (not ROW__IS__DELETED)) (type: boolean) + expressions: a (type: int), c (type: decimal(10,2)), ROW__ID (type: struct), (ROW__IS__DELETED and (ROW__ID.writeid <= 1L) is not true) (type: boolean), ((ROW__ID.writeid <= 1L) is not true and (not ROW__IS__DELETED)) (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator @@ -278,7 +278,7 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 4 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: int), c (type: decimal(10,2)), ROW__ID (type: struct), (ROW__IS__DELETED and (ROW__ID.writeid > 1L)) (type: boolean), ((ROW__ID.writeid > 1L) and (not ROW__IS__DELETED)) (type: boolean) + expressions: a (type: int), c (type: decimal(10,2)), ROW__ID (type: struct), (ROW__IS__DELETED and (ROW__ID.writeid <= 1L) is not true) (type: boolean), ((ROW__ID.writeid <= 1L) is not true and (not ROW__IS__DELETED)) (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 4 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator @@ -302,7 +302,7 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: int), ROW__ID (type: struct), (ROW__IS__DELETED and (ROW__ID.writeid > 1L)) (type: boolean), ((ROW__ID.writeid > 1L) and (not ROW__IS__DELETED)) (type: boolean) + expressions: a (type: int), ROW__ID (type: struct), (ROW__IS__DELETED and (ROW__ID.writeid <= 1L) is not true) (type: boolean), ((ROW__ID.writeid <= 1L) is not true and (not ROW__IS__DELETED)) (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 4 Data size: 352 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator @@ -528,7 +528,7 @@ STAGE PLANS: 0 _col4 (type: int) 1 _col0 (type: int) outputColumnNames: _col1, _col2, _col3, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - residual filter predicates: {((_col10 > 10) or (_col5 > 10))} {(((not _col12) and (not _col7)) or ((not _col13) and (not _col8)))} {(((not _col12) and (not _col7) and (not _col2)) or ((not _col13) and (not _col8) and (not _col3)))} {((_col6.writeid > 1L) or (_col11.writeid > 1L) or (_col1.writeid > 1L))} + residual filter predicates: {((_col10 > 10) or (_col5 > 10))} {(((not _col12) and (not _col7)) or ((not _col13) and (not _col8)))} {(((not _col12) and (not _col7) and (not _col2)) or ((not _col13) and (not _col8) and (not _col3)))} {((_col6.writeid <= 1L) and (_col11.writeid <= 1L) and (_col1.writeid <= 1L)) is not true} Statistics: Num rows: 1 Data size: 480 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col9 (type: int), _col5 (type: decimal(10,2)), if((_col12 or _col7 or _col2), -1, 1) (type: int) @@ -696,8 +696,8 @@ HiveProject(cmv_basetable_n6.a=[$4], cmv_basetable_2_n3.c=[$5], _c2=[CASE(IS NUL HiveProject(a=[$0], c=[$2]) HiveFilter(condition=[IS NOT NULL($0)]) HiveTableScan(table=[[default, cmv_basetable_2_n3]], table:alias=[cmv_basetable_2_n3]) - HiveProject(a=[$0], c=[$2], _deleted=[AND($7, IS NOT NULL($6.writeid))]) - HiveFilter(condition=[AND(<(2, $6.writeid), IS NOT NULL($7), IS NOT NULL($0))]) + HiveProject(a=[$0], c=[$2], _deleted=[AND($7, IS NOT TRUE(>=(2, $6.writeid)))]) + HiveFilter(condition=[AND(OR(>=(2, $6.writeid), IS NOT NULL($7)), IS NOT TRUE(>=(2, $6.writeid)), IS NOT NULL($0))]) HiveTableScan(table=[[default, cmv_basetable_n6]], table:alias=[cmv_basetable_n6]) PREHOOK: query: EXPLAIN @@ -768,15 +768,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cmv_basetable_n6 - filterExpr: ((ROW__ID.writeid > 2L) and ROW__IS__DELETED is not null and a is not null) (type: boolean) + filterExpr: (((ROW__ID.writeid <= 2L) or ROW__IS__DELETED is not null) and (ROW__ID.writeid <= 2L) is not true and a is not null) (type: boolean) properties: acid.fetch.deleted.rows TRUE Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((ROW__ID.writeid > 2L) and ROW__IS__DELETED is not null and a is not null) (type: boolean) + predicate: (((ROW__ID.writeid <= 2L) or ROW__IS__DELETED is not null) and (ROW__ID.writeid <= 2L) is not true and a is not null) (type: boolean) Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: int), c (type: decimal(10,2)), (ROW__IS__DELETED and ROW__ID.writeid is not null) (type: boolean) + expressions: a (type: int), c (type: decimal(10,2)), (ROW__IS__DELETED and (ROW__ID.writeid <= 2L) is not true) (type: boolean) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_9.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_9.q.out index 0764a8ed1d70..5e104a7c3d44 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_9.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_9.q.out @@ -253,11 +253,11 @@ HiveProject(t1.a=[$4], _c1=[CASE(IS NULL($1), $5, IS NULL($5), $1, +($5, $1))], HiveProject(a=[$0], $f1=[$1], $f2=[$2]) HiveAggregate(group=[{0}], agg#0=[SUM($1)], agg#1=[SUM($2)]) HiveProject(a=[$0], $f4=[CASE(OR($3, $7), *(-1, $1), $1)], $f5=[CASE(OR($3, $7), -1, 1)]) - HiveJoin(condition=[AND(=($0, $5), OR(AND(NOT($3), NOT($7)), AND(NOT($4), NOT($8))), OR(<(3, $2.writeid), <(3, $6.writeid)))], joinType=[inner], algorithm=[none], cost=[not available]) - HiveProject(a=[$0], b=[$1], ROW__ID=[$5], _deleted=[AND($6, <(3, $5.writeid))], _inserted=[AND(<(3, $5.writeid), NOT($6))]) + HiveJoin(condition=[AND(=($0, $5), OR(AND(NOT($3), NOT($7)), AND(NOT($4), NOT($8))), IS NOT TRUE(AND(>=(3, $2.writeid), >=(3, $6.writeid))))], joinType=[inner], algorithm=[none], cost=[not available]) + HiveProject(a=[$0], b=[$1], ROW__ID=[$5], _deleted=[AND($6, IS NOT TRUE(>=(3, $5.writeid)))], _inserted=[AND(IS NOT TRUE(>=(3, $5.writeid)), NOT($6))]) HiveFilter(condition=[IS NOT NULL($0)]) HiveTableScan(table=[[default, t1]], table:alias=[t1]) - HiveProject(a=[$0], ROW__ID=[$4], _deleted=[AND($5, <(3, $4.writeid))], _inserted=[AND(<(3, $4.writeid), NOT($5))]) + HiveProject(a=[$0], ROW__ID=[$4], _deleted=[AND($5, IS NOT TRUE(>=(3, $4.writeid)))], _inserted=[AND(IS NOT TRUE(>=(3, $4.writeid)), NOT($5))]) HiveFilter(condition=[IS NOT NULL($0)]) HiveTableScan(table=[[default, t2]], table:alias=[t2]) @@ -334,7 +334,7 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 9 Data size: 837 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: char(15)), ROW__ID (type: struct), (ROW__IS__DELETED and (ROW__ID.writeid > 3L)) (type: boolean), ((ROW__ID.writeid > 3L) and (not ROW__IS__DELETED)) (type: boolean) + expressions: a (type: char(15)), ROW__ID (type: struct), (ROW__IS__DELETED and (ROW__ID.writeid <= 3L) is not true) (type: boolean), ((ROW__ID.writeid <= 3L) is not true and (not ROW__IS__DELETED)) (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 9 Data size: 1593 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator @@ -358,7 +358,7 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 7 Data size: 671 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: char(15)), b (type: int), ROW__ID (type: struct), (ROW__IS__DELETED and (ROW__ID.writeid > 3L)) (type: boolean), ((ROW__ID.writeid > 3L) and (not ROW__IS__DELETED)) (type: boolean) + expressions: a (type: char(15)), b (type: int), ROW__ID (type: struct), (ROW__IS__DELETED and (ROW__ID.writeid <= 3L) is not true) (type: boolean), ((ROW__ID.writeid <= 3L) is not true and (not ROW__IS__DELETED)) (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 7 Data size: 1259 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator @@ -550,7 +550,7 @@ STAGE PLANS: 0 _col0 (type: char(15)) 1 _col0 (type: char(15)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col6, _col7, _col8 - residual filter predicates: {(((not _col3) and (not _col7)) or ((not _col4) and (not _col8)))} {((_col2.writeid > 3L) or (_col6.writeid > 3L))} + residual filter predicates: {(((not _col3) and (not _col7)) or ((not _col4) and (not _col8)))} {((_col2.writeid <= 3L) and (_col6.writeid <= 3L)) is not true} Statistics: Num rows: 1 Data size: 265 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: char(15)), if((_col3 or _col7), (-1 * _col1), _col1) (type: int), if((_col3 or _col7), -1, 1) (type: int) From 5c84b27c1b2c2a9ad87213d67e8da5e1d0ef0880 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Tue, 8 Sep 2026 11:37:37 +0100 Subject: [PATCH 15/24] Update json query88.q.out (simplification due to CALCITE-7722) --- .../perf/tpcds30tb/json/query88.q.out | 320 ++++-------------- 1 file changed, 72 insertions(+), 248 deletions(-) diff --git a/ql/src/test/results/clientpositive/perf/tpcds30tb/json/query88.q.out b/ql/src/test/results/clientpositive/perf/tpcds30tb/json/query88.q.out index 463ea7d44ff5..1817971b4306 100644 --- a/ql/src/test/results/clientpositive/perf/tpcds30tb/json/query88.q.out +++ b/ql/src/test/results/clientpositive/perf/tpcds30tb/json/query88.q.out @@ -862,43 +862,21 @@ Warning: Map Join MAPJOIN[599][bigTable=?] in task 'Reducer 5' is a cross produc "operands": [ { "op": { - "name": "AND", - "kind": "AND", + "name": "=", + "kind": "EQUALS", "syntax": "BINARY" }, "operands": [ { - "op": { - "name": "=", - "kind": "EQUALS", - "syntax": "BINARY" - }, - "operands": [ - { - "input": 3, - "name": "$3" - }, - { - "literal": 3, - "type": { - "type": "INTEGER", - "nullable": false - } - } - ] + "input": 3, + "name": "$3" }, { - "op": { - "name": "IS NOT NULL", - "kind": "IS_NOT_NULL", - "syntax": "POSTFIX" - }, - "operands": [ - { - "input": 4, - "name": "$4" - } - ] + "literal": 3, + "type": { + "type": "INTEGER", + "nullable": false + } } ] }, @@ -1768,43 +1746,21 @@ Warning: Map Join MAPJOIN[599][bigTable=?] in task 'Reducer 5' is a cross produc "operands": [ { "op": { - "name": "AND", - "kind": "AND", + "name": "=", + "kind": "EQUALS", "syntax": "BINARY" }, "operands": [ { - "op": { - "name": "=", - "kind": "EQUALS", - "syntax": "BINARY" - }, - "operands": [ - { - "input": 3, - "name": "$3" - }, - { - "literal": 3, - "type": { - "type": "INTEGER", - "nullable": false - } - } - ] + "input": 3, + "name": "$3" }, { - "op": { - "name": "IS NOT NULL", - "kind": "IS_NOT_NULL", - "syntax": "POSTFIX" - }, - "operands": [ - { - "input": 4, - "name": "$4" - } - ] + "literal": 3, + "type": { + "type": "INTEGER", + "nullable": false + } } ] }, @@ -2338,43 +2294,21 @@ Warning: Map Join MAPJOIN[599][bigTable=?] in task 'Reducer 5' is a cross produc "operands": [ { "op": { - "name": "AND", - "kind": "AND", + "name": "=", + "kind": "EQUALS", "syntax": "BINARY" }, "operands": [ { - "op": { - "name": "=", - "kind": "EQUALS", - "syntax": "BINARY" - }, - "operands": [ - { - "input": 3, - "name": "$3" - }, - { - "literal": 3, - "type": { - "type": "INTEGER", - "nullable": false - } - } - ] + "input": 3, + "name": "$3" }, { - "op": { - "name": "IS NOT NULL", - "kind": "IS_NOT_NULL", - "syntax": "POSTFIX" - }, - "operands": [ - { - "input": 4, - "name": "$4" - } - ] + "literal": 3, + "type": { + "type": "INTEGER", + "nullable": false + } } ] }, @@ -2908,43 +2842,21 @@ Warning: Map Join MAPJOIN[599][bigTable=?] in task 'Reducer 5' is a cross produc "operands": [ { "op": { - "name": "AND", - "kind": "AND", + "name": "=", + "kind": "EQUALS", "syntax": "BINARY" }, "operands": [ { - "op": { - "name": "=", - "kind": "EQUALS", - "syntax": "BINARY" - }, - "operands": [ - { - "input": 3, - "name": "$3" - }, - { - "literal": 3, - "type": { - "type": "INTEGER", - "nullable": false - } - } - ] + "input": 3, + "name": "$3" }, { - "op": { - "name": "IS NOT NULL", - "kind": "IS_NOT_NULL", - "syntax": "POSTFIX" - }, - "operands": [ - { - "input": 4, - "name": "$4" - } - ] + "literal": 3, + "type": { + "type": "INTEGER", + "nullable": false + } } ] }, @@ -3478,43 +3390,21 @@ Warning: Map Join MAPJOIN[599][bigTable=?] in task 'Reducer 5' is a cross produc "operands": [ { "op": { - "name": "AND", - "kind": "AND", + "name": "=", + "kind": "EQUALS", "syntax": "BINARY" }, "operands": [ { - "op": { - "name": "=", - "kind": "EQUALS", - "syntax": "BINARY" - }, - "operands": [ - { - "input": 3, - "name": "$3" - }, - { - "literal": 3, - "type": { - "type": "INTEGER", - "nullable": false - } - } - ] + "input": 3, + "name": "$3" }, { - "op": { - "name": "IS NOT NULL", - "kind": "IS_NOT_NULL", - "syntax": "POSTFIX" - }, - "operands": [ - { - "input": 4, - "name": "$4" - } - ] + "literal": 3, + "type": { + "type": "INTEGER", + "nullable": false + } } ] }, @@ -4048,43 +3938,21 @@ Warning: Map Join MAPJOIN[599][bigTable=?] in task 'Reducer 5' is a cross produc "operands": [ { "op": { - "name": "AND", - "kind": "AND", + "name": "=", + "kind": "EQUALS", "syntax": "BINARY" }, "operands": [ { - "op": { - "name": "=", - "kind": "EQUALS", - "syntax": "BINARY" - }, - "operands": [ - { - "input": 3, - "name": "$3" - }, - { - "literal": 3, - "type": { - "type": "INTEGER", - "nullable": false - } - } - ] + "input": 3, + "name": "$3" }, { - "op": { - "name": "IS NOT NULL", - "kind": "IS_NOT_NULL", - "syntax": "POSTFIX" - }, - "operands": [ - { - "input": 4, - "name": "$4" - } - ] + "literal": 3, + "type": { + "type": "INTEGER", + "nullable": false + } } ] }, @@ -4618,43 +4486,21 @@ Warning: Map Join MAPJOIN[599][bigTable=?] in task 'Reducer 5' is a cross produc "operands": [ { "op": { - "name": "AND", - "kind": "AND", + "name": "=", + "kind": "EQUALS", "syntax": "BINARY" }, "operands": [ { - "op": { - "name": "=", - "kind": "EQUALS", - "syntax": "BINARY" - }, - "operands": [ - { - "input": 3, - "name": "$3" - }, - { - "literal": 3, - "type": { - "type": "INTEGER", - "nullable": false - } - } - ] + "input": 3, + "name": "$3" }, { - "op": { - "name": "IS NOT NULL", - "kind": "IS_NOT_NULL", - "syntax": "POSTFIX" - }, - "operands": [ - { - "input": 4, - "name": "$4" - } - ] + "literal": 3, + "type": { + "type": "INTEGER", + "nullable": false + } } ] }, @@ -5188,43 +5034,21 @@ Warning: Map Join MAPJOIN[599][bigTable=?] in task 'Reducer 5' is a cross produc "operands": [ { "op": { - "name": "AND", - "kind": "AND", + "name": "=", + "kind": "EQUALS", "syntax": "BINARY" }, "operands": [ { - "op": { - "name": "=", - "kind": "EQUALS", - "syntax": "BINARY" - }, - "operands": [ - { - "input": 3, - "name": "$3" - }, - { - "literal": 3, - "type": { - "type": "INTEGER", - "nullable": false - } - } - ] + "input": 3, + "name": "$3" }, { - "op": { - "name": "IS NOT NULL", - "kind": "IS_NOT_NULL", - "syntax": "POSTFIX" - }, - "operands": [ - { - "input": 4, - "name": "$4" - } - ] + "literal": 3, + "type": { + "type": "INTEGER", + "nullable": false + } } ] }, From 217b825f13077b9b2e24b41a3c12bb3cc957db4f Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Tue, 8 Sep 2026 13:20:11 +0100 Subject: [PATCH 16/24] Adjust test files 'ROW__ID.writeid > 1L' ==> '(ROW__ID.writeid <= 1L) is not true' due to CALCITE-7636 --- .../src/test/results/positive/mv_iceberg_orc8.q.out | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc8.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc8.q.out index 23948e8892ca..e1999f4cfb79 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc8.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc8.q.out @@ -102,10 +102,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1 - filterExpr: (ROW__ID.writeid > 1L) (type: boolean) + filterExpr: (ROW__ID.writeid <= 1L) is not true (type: boolean) Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (ROW__ID.writeid > 1L) (type: boolean) + predicate: (ROW__ID.writeid <= 1L) is not true (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), b (type: varchar(256)), c (type: char(100)), d (type: int) From 2a3fead53ad3610c7980a95850d3eea0a5ca97fc Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Wed, 9 Sep 2026 08:59:11 +0100 Subject: [PATCH 17/24] Adjust test plan explainuser_1.q.out (simplifications probably from CALCITE-7722) --- .../clientpositive/llap/explainuser_1.q.out | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out index fcde373c805c..4e6c1d4bdc90 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -555,21 +555,21 @@ Stage-0 SHUFFLE [RS_25] PartitionCols:_col0, _col1 Group By Operator [GBY_24] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col1, _col7 + Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col1, _col6 Select Operator [SEL_23] (rows=1 width=20) - Output:["_col1","_col7"] + Output:["_col1","_col6"] Merge Join Operator [MERGEJOIN_64] (rows=1 width=20) - Conds:RS_20._col5=RS_21._col0(Inner),Output:["_col1","_col6","_col7","_col9"],residual filter predicates:{((_col6 + _col9) >= 0)} + Conds:RS_20._col4=RS_21._col0(Inner),Output:["_col1","_col5","_col6","_col8"],residual filter predicates:{((_col5 + _col8) >= 0)} <-Reducer 2 [SIMPLE_EDGE] llap SHUFFLE [RS_20] - PartitionCols:_col5 - Merge Join Operator [MERGEJOIN_63] (rows=1 width=117) - Conds:RS_17._col0=RS_18._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7"],residual filter predicates:{((_col6 > 0) or _col2)} {(_col3 or (_col7 >= 1L))} {((_col4 + _col7) >= 0L)} + PartitionCols:_col4 + Merge Join Operator [MERGEJOIN_63] (rows=2 width=113) + Conds:RS_17._col0=RS_18._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"],residual filter predicates:{(_col2 or (_col6 >= 1L))} {((_col3 + _col6) >= 0L)} <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_17] PartitionCols:_col0 - Select Operator [SEL_2] (rows=18 width=99) - Output:["_col0","_col1","_col2","_col3","_col4"] + Select Operator [SEL_2] (rows=18 width=95) + Output:["_col0","_col1","_col2","_col3"] Filter Operator [FIL_37] (rows=18 width=84) predicate:((c_int > 0) and key is not null) TableScan [TS_0] (rows=20 width=84) @@ -587,7 +587,7 @@ Stage-0 Group By Operator [GBY_6] (rows=2 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_38] (rows=2 width=93) - predicate:((c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((c_int + 1) >= 0) and ((UDFToFloat(c_int) + c_float) >= 0.0) and ((c_int > 0) or c_float is not null) and key is not null) + predicate:((c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((c_int + 1) >= 0) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 9 [SIMPLE_EDGE] llap @@ -603,7 +603,7 @@ Stage-0 Group By Operator [GBY_13] (rows=2 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_39] (rows=2 width=93) - predicate:((c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((c_int + 1) >= 0) and ((UDFToFloat(c_int) + c_float) >= 0.0) and ((c_int > 0) or c_float is not null) and key is not null) + predicate:((c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((c_int + 1) >= 0) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -676,7 +676,7 @@ Stage-0 Group By Operator [GBY_6] (rows=2 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_35] (rows=2 width=93) - predicate:((c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((c_int + 1) >= 0) and ((UDFToFloat(c_int) + c_float) >= 0.0) and ((c_int > 0) or c_float is not null) and key is not null) + predicate:((c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((c_int + 1) >= 0) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 8 [SIMPLE_EDGE] llap @@ -692,7 +692,7 @@ Stage-0 Group By Operator [GBY_13] (rows=2 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_36] (rows=2 width=93) - predicate:((c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((c_int + 1) >= 0) and ((UDFToFloat(c_int) + c_float) >= 0.0) and ((c_int > 0) or c_float is not null) and key is not null) + predicate:((c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((c_int + 1) >= 0) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -738,21 +738,21 @@ Stage-0 SHUFFLE [RS_25] PartitionCols:_col0, _col1 Group By Operator [GBY_24] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col1, _col7 + Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col1, _col6 Select Operator [SEL_23] (rows=1 width=20) - Output:["_col1","_col7"] + Output:["_col1","_col6"] Merge Join Operator [MERGEJOIN_63] (rows=1 width=20) - Conds:RS_20._col5=RS_21._col0(Inner),Output:["_col1","_col6","_col7","_col9"],residual filter predicates:{((_col6 + _col9) >= 0)} + Conds:RS_20._col4=RS_21._col0(Inner),Output:["_col1","_col5","_col6","_col8"],residual filter predicates:{((_col5 + _col8) >= 0)} <-Reducer 2 [SIMPLE_EDGE] llap SHUFFLE [RS_20] - PartitionCols:_col5 - Merge Join Operator [MERGEJOIN_62] (rows=1 width=117) - Conds:RS_17._col0=RS_18._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7"],residual filter predicates:{((_col6 > 0) or _col2)} {(_col3 or (_col7 >= 1L))} {((_col4 + _col7) >= 0L)} + PartitionCols:_col4 + Merge Join Operator [MERGEJOIN_62] (rows=2 width=113) + Conds:RS_17._col0=RS_18._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"],residual filter predicates:{(_col2 or (_col6 >= 1L))} {((_col3 + _col6) >= 0L)} <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_17] PartitionCols:_col0 - Select Operator [SEL_2] (rows=18 width=99) - Output:["_col0","_col1","_col2","_col3","_col4"] + Select Operator [SEL_2] (rows=18 width=95) + Output:["_col0","_col1","_col2","_col3"] Filter Operator [FIL_36] (rows=18 width=84) predicate:((c_int > 0) and key is not null) TableScan [TS_0] (rows=20 width=84) @@ -770,7 +770,7 @@ Stage-0 Group By Operator [GBY_6] (rows=2 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_37] (rows=2 width=93) - predicate:((c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((c_int + 1) >= 0) and ((UDFToFloat(c_int) + c_float) >= 0.0) and ((c_int > 0) or c_float is not null) and key is not null) + predicate:((c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((c_int + 1) >= 0) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 9 [SIMPLE_EDGE] llap @@ -786,7 +786,7 @@ Stage-0 Group By Operator [GBY_13] (rows=2 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_38] (rows=2 width=93) - predicate:((c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((c_int + 1) >= 0) and ((UDFToFloat(c_int) + c_float) >= 0.0) and ((c_int > 0) or c_float is not null) and key is not null) + predicate:((c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((c_int + 1) >= 0) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -859,7 +859,7 @@ Stage-0 Group By Operator [GBY_6] (rows=2 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_35] (rows=2 width=93) - predicate:((c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((c_int + 1) >= 0) and ((UDFToFloat(c_int) + c_float) >= 0.0) and ((c_int > 0) or c_float is not null) and key is not null) + predicate:((c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((c_int + 1) >= 0) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 8 [SIMPLE_EDGE] llap @@ -875,7 +875,7 @@ Stage-0 Group By Operator [GBY_13] (rows=2 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_36] (rows=2 width=93) - predicate:((c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((c_int + 1) >= 0) and ((UDFToFloat(c_int) + c_float) >= 0.0) and ((c_int > 0) or c_float is not null) and key is not null) + predicate:((c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((c_int + 1) >= 0) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -1924,7 +1924,7 @@ Stage-0 Group By Operator [GBY_3] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_42] (rows=1 width=93) - predicate:((c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((c_int + 1) >= 0) and ((UDFToFloat(c_int) + c_float) >= 0.0) and (((c_int + 1) + 1) >= 0) and (UDFToDouble(key) > 0.0D) and ((c_int > 0) or c_float is not null) and (((c_int + 1) > 0) or UDFToDouble(key) is not null)) + predicate:((c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((c_int + 1) >= 0) and ((UDFToFloat(c_int) + c_float) >= 0.0) and (((c_int + 1) + 1) >= 0) and (UDFToDouble(key) > 0.0D)) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 8 [SIMPLE_EDGE] llap @@ -1942,7 +1942,7 @@ Stage-0 Group By Operator [GBY_12] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_43] (rows=1 width=93) - predicate:((c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((c_int + 1) >= 0) and ((UDFToFloat(c_int) + c_float) >= 0.0) and (UDFToDouble(key) > 0.0D) and ((c_int > 0) or c_float is not null)) + predicate:((c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((c_int + 1) >= 0) and ((UDFToFloat(c_int) + c_float) >= 0.0) and (UDFToDouble(key) > 0.0D)) TableScan [TS_9] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] From 27f5ecc3463d3f5484866a7f0875a7923a1f456c Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Wed, 9 Sep 2026 09:03:03 +0100 Subject: [PATCH 18/24] Adjust test file vector_case_when_2.q.out: probably due to CALCITE-7529 TIMESTAMP literals are now preserved as TimestampString at declared precision instead of round-tripping through millisecond-precision runtime values, so zero-fractional literals now render with a full .000000000 nanosecond suffix (semantically identical) --- .../llap/vector_case_when_2.q.out | 246 +++++++++--------- 1 file changed, 123 insertions(+), 123 deletions(-) diff --git a/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out b/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out index 142ce776180f..d4965bb13c29 100644 --- a/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out @@ -144,14 +144,14 @@ STAGE PLANS: native: true vectorizationSchemaColumns: [0:cdate:date, 1:ctimestamp1:timestamp, 2:stimestamp1:string, 3:ctimestamp2:timestamp, 4:ROW__ID:struct, 5:ROW__IS__DELETED:boolean] Select Operator - expressions: ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), CASE WHEN ((ctimestamp2 <= TIMESTAMP'1800-12-31 00:00:00')) THEN ('1800s or Earlier') WHEN ((ctimestamp2 < TIMESTAMP'1900-01-01 00:00:00')) THEN ('1900s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN ('Early 2010s') ELSE ('Unknown') END (type: string), CASE WHEN ((ctimestamp2 <= TIMESTAMP'2000-12-31 23:59:59.999999999')) THEN ('Old') WHEN ((ctimestamp2 < TIMESTAMP'2006-01-01 00:00:00')) THEN ('Early 2000s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN ('Early 2010s') ELSE (null) END (type: string), CASE WHEN ((ctimestamp2 <= TIMESTAMP'2000-12-31 23:59:59.999999999')) THEN ('Old') WHEN ((ctimestamp2 < TIMESTAMP'2006-01-01 00:00:00')) THEN ('Early 2000s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') ELSE (null) END (type: string), if((ctimestamp1 < TIMESTAMP'1974-10-04 17:21:03.989'), year(ctimestamp1), year(ctimestamp2)) (type: int), if((stimestamp1 like '%19%'), stimestamp1, '2018-03-08 23:04:59') (type: string), if((ctimestamp1 = TIMESTAMP'2021-09-24 03:18:32.413655165'), null, minute(ctimestamp1)) (type: int), if(((ctimestamp2 >= TIMESTAMP'5344-10-04 18:40:08.165') and (ctimestamp2 < TIMESTAMP'6631-11-13 16:31:29.702202248')), minute(ctimestamp1), null) (type: int), if(((UDFToDouble(ctimestamp1) % 500.0D) > 100.0D), date_add(cdate, 1), date_add(cdate, 365)) (type: date), stimestamp1 (type: string) + expressions: ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), CASE WHEN ((ctimestamp2 <= TIMESTAMP'1800-12-31 00:00:00')) THEN ('1800s or Earlier') WHEN ((ctimestamp2 < TIMESTAMP'1900-01-01 00:00:00')) THEN ('1900s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN ('Early 2010s') ELSE ('Unknown') END (type: string), CASE WHEN ((ctimestamp2 <= TIMESTAMP'2000-12-31 23:59:59.999999999')) THEN ('Old') WHEN ((ctimestamp2 < TIMESTAMP'2006-01-01 00:00:00')) THEN ('Early 2000s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN ('Early 2010s') ELSE (null) END (type: string), CASE WHEN ((ctimestamp2 <= TIMESTAMP'2000-12-31 23:59:59.999999999')) THEN ('Old') WHEN ((ctimestamp2 < TIMESTAMP'2006-01-01 00:00:00')) THEN ('Early 2000s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') ELSE (null) END (type: string), if((ctimestamp1 < TIMESTAMP'1974-10-04 17:21:03.989'), year(ctimestamp1), year(ctimestamp2)) (type: int), if((stimestamp1 like '%19%'), stimestamp1, '2018-03-08 23:04:59.000000000') (type: string), if((ctimestamp1 = TIMESTAMP'2021-09-24 03:18:32.413655165'), null, minute(ctimestamp1)) (type: int), if(((ctimestamp2 >= TIMESTAMP'5344-10-04 18:40:08.165') and (ctimestamp2 < TIMESTAMP'6631-11-13 16:31:29.702202248')), minute(ctimestamp1), null) (type: int), if(((UDFToDouble(ctimestamp1) % 500.0D) > 100.0D), date_add(cdate, 1), date_add(cdate, 365)) (type: date), stimestamp1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 Select Vectorization: className: VectorSelectOperator native: true projectedOutputColumnNums: [1, 3, 10, 15, 19, 23, 25, 28, 33, 39, 2] - selectExpressions: VectorUDFAdaptor(CASE WHEN ((ctimestamp2 <= TIMESTAMP'1800-12-31 00:00:00')) THEN ('1800s or Earlier') WHEN ((ctimestamp2 < TIMESTAMP'1900-01-01 00:00:00')) THEN ('1900s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN ('Early 2010s') ELSE ('Unknown') END)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 1800-12-31 00:00:00) -> 6:boolean, TimestampColLessTimestampScalar(col 3:timestamp, val 1900-01-01 00:00:00) -> 7:boolean, TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 8:boolean, TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 9:boolean) -> 10:string, VectorUDFAdaptor(CASE WHEN ((ctimestamp2 <= TIMESTAMP'2000-12-31 23:59:59.999999999')) THEN ('Old') WHEN ((ctimestamp2 < TIMESTAMP'2006-01-01 00:00:00')) THEN ('Early 2000s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN ('Early 2010s') ELSE (null) END)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2000-12-31 23:59:59.999999999) -> 11:boolean, TimestampColLessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 12:boolean, TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 13:boolean, TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 14:boolean) -> 15:string, VectorUDFAdaptor(CASE WHEN ((ctimestamp2 <= TIMESTAMP'2000-12-31 23:59:59.999999999')) THEN ('Old') WHEN ((ctimestamp2 < TIMESTAMP'2006-01-01 00:00:00')) THEN ('Early 2000s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') ELSE (null) END)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2000-12-31 23:59:59.999999999) -> 16:boolean, TimestampColLessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 17:boolean, TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 18:boolean) -> 19:string, IfExprLongColumnLongColumn(col 20:boolean, col 21:int, col 22:int)(children: TimestampColLessTimestampScalar(col 1:timestamp, val 1974-10-04 17:21:03.989) -> 20:boolean, VectorUDFYearTimestamp(col 1:timestamp, field YEAR) -> 21:int, VectorUDFYearTimestamp(col 3:timestamp, field YEAR) -> 22:int) -> 23:int, IfExprStringGroupColumnStringScalar(col 24:boolean, col 2:string, val 2018-03-08 23:04:59)(children: SelectStringColLikeStringScalar(col 2:string) -> 24:boolean) -> 25:string, VectorUDFAdaptor(if((ctimestamp1 = TIMESTAMP'2021-09-24 03:18:32.413655165'), null, minute(ctimestamp1)))(children: TimestampColEqualTimestampScalar(col 1:timestamp, val 2021-09-24 03:18:32.413655165) -> 26:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 27:int) -> 28:int, VectorUDFAdaptor(if(((ctimestamp2 >= TIMESTAMP'5344-10-04 18:40:08.165') and (ctimestamp2 < TIMESTAMP'6631-11-13 16:31:29.702202248')), minute(ctimestamp1), null))(children: ColAndCol(col 29:boolean, col 30:boolean)(children: TimestampColGreaterEqualTimestampScalar(col 3:timestamp, val 5344-10-04 18:40:08.165) -> 29:boolean, TimestampColLessTimestampScalar(col 3:timestamp, val 6631-11-13 16:31:29.702202248) -> 30:boolean) -> 31:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 32:int) -> 33:int, IfExprLongColumnLongColumn(col 36:boolean, col 37:date, col 38:date)(children: DoubleColGreaterDoubleScalar(col 35:double, val 100.0)(children: DoubleColModuloDoubleScalar(col 34:double, val 500.0)(children: CastTimestampToDouble(col 1:timestamp) -> 34:double) -> 35:double) -> 36:boolean, VectorUDFDateAddColScalar(col 0:date, val 1) -> 37:date, VectorUDFDateAddColScalar(col 0:date, val 365) -> 38:date) -> 39:date - Statistics: Num rows: 51 Data size: 23760 Basic stats: COMPLETE Column stats: COMPLETE + selectExpressions: VectorUDFAdaptor(CASE WHEN ((ctimestamp2 <= TIMESTAMP'1800-12-31 00:00:00')) THEN ('1800s or Earlier') WHEN ((ctimestamp2 < TIMESTAMP'1900-01-01 00:00:00')) THEN ('1900s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN ('Early 2010s') ELSE ('Unknown') END)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 1800-12-31 00:00:00) -> 6:boolean, TimestampColLessTimestampScalar(col 3:timestamp, val 1900-01-01 00:00:00) -> 7:boolean, TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 8:boolean, TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 9:boolean) -> 10:string, VectorUDFAdaptor(CASE WHEN ((ctimestamp2 <= TIMESTAMP'2000-12-31 23:59:59.999999999')) THEN ('Old') WHEN ((ctimestamp2 < TIMESTAMP'2006-01-01 00:00:00')) THEN ('Early 2000s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN ('Early 2010s') ELSE (null) END)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2000-12-31 23:59:59.999999999) -> 11:boolean, TimestampColLessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 12:boolean, TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 13:boolean, TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 14:boolean) -> 15:string, VectorUDFAdaptor(CASE WHEN ((ctimestamp2 <= TIMESTAMP'2000-12-31 23:59:59.999999999')) THEN ('Old') WHEN ((ctimestamp2 < TIMESTAMP'2006-01-01 00:00:00')) THEN ('Early 2000s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') ELSE (null) END)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2000-12-31 23:59:59.999999999) -> 16:boolean, TimestampColLessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 17:boolean, TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 18:boolean) -> 19:string, IfExprLongColumnLongColumn(col 20:boolean, col 21:int, col 22:int)(children: TimestampColLessTimestampScalar(col 1:timestamp, val 1974-10-04 17:21:03.989) -> 20:boolean, VectorUDFYearTimestamp(col 1:timestamp, field YEAR) -> 21:int, VectorUDFYearTimestamp(col 3:timestamp, field YEAR) -> 22:int) -> 23:int, IfExprStringGroupColumnStringScalar(col 24:boolean, col 2:string, val 2018-03-08 23:04:59.000000000)(children: SelectStringColLikeStringScalar(col 2:string) -> 24:boolean) -> 25:string, VectorUDFAdaptor(if((ctimestamp1 = TIMESTAMP'2021-09-24 03:18:32.413655165'), null, minute(ctimestamp1)))(children: TimestampColEqualTimestampScalar(col 1:timestamp, val 2021-09-24 03:18:32.413655165) -> 26:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 27:int) -> 28:int, VectorUDFAdaptor(if(((ctimestamp2 >= TIMESTAMP'5344-10-04 18:40:08.165') and (ctimestamp2 < TIMESTAMP'6631-11-13 16:31:29.702202248')), minute(ctimestamp1), null))(children: ColAndCol(col 29:boolean, col 30:boolean)(children: TimestampColGreaterEqualTimestampScalar(col 3:timestamp, val 5344-10-04 18:40:08.165) -> 29:boolean, TimestampColLessTimestampScalar(col 3:timestamp, val 6631-11-13 16:31:29.702202248) -> 30:boolean) -> 31:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 32:int) -> 33:int, IfExprLongColumnLongColumn(col 36:boolean, col 37:date, col 38:date)(children: DoubleColGreaterDoubleScalar(col 35:double, val 100.0)(children: DoubleColModuloDoubleScalar(col 34:double, val 500.0)(children: CastTimestampToDouble(col 1:timestamp) -> 34:double) -> 35:double) -> 36:boolean, VectorUDFDateAddColScalar(col 0:date, val 1) -> 37:date, VectorUDFDateAddColScalar(col 0:date, val 365) -> 38:date) -> 39:date + Statistics: Num rows: 51 Data size: 23862 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: timestamp), _col10 (type: string), _col1 (type: timestamp) null sort order: zzz @@ -162,7 +162,7 @@ STAGE PLANS: native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true valueColumns: 10:string, 15:string, 19:string, 23:int, 25:string, 28:int, 33:int, 39:date - Statistics: Num rows: 51 Data size: 23760 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 51 Data size: 23862 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int), _col8 (type: int), _col9 (type: date) Execution mode: vectorized, llap LLAP IO: all inputs @@ -204,13 +204,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 2, 3, 4, 5, 6, 7, 8, 9, 10] - Statistics: Num rows: 51 Data size: 18099 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 51 Data size: 18201 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 51 Data size: 18099 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 51 Data size: 18201 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -290,11 +290,11 @@ POSTHOOK: Input: default@timestamps #### A masked pattern was here #### ctimestamp1 ctimestamp2 ctimestamp2_description ctimestamp2_description_2 ctimestamp2_description_3 field1 field_2 field_3 field_4 field_5 0004-09-22 18:26:29.519542222 0004-09-21 16:23:25.519542222 1800s or Earlier Old Old 4 0004-09-22 18:26:29.519542222 26 NULL 0005-09-22 -0528-10-27 08:15:18.941718273 0528-10-26 06:12:14.941718273 1800s or Earlier Old Old 528 2018-03-08 23:04:59 15 NULL 0529-10-27 +0528-10-27 08:15:18.941718273 0528-10-26 06:12:14.941718273 1800s or Earlier Old Old 528 2018-03-08 23:04:59.000000000 15 NULL 0529-10-27 1319-02-02 16:31:57.778 1319-02-01 14:28:53.778 1800s or Earlier Old Old 1319 1319-02-02 16:31:57.778 31 NULL 1320-02-02 -1404-07-23 15:32:16.059185026 1404-07-22 13:29:12.059185026 1800s or Earlier Old Old 1404 2018-03-08 23:04:59 32 NULL 1405-07-23 -1815-05-06 00:12:37.543584705 1815-05-04 22:09:33.543584705 1900s Old Old 1815 2018-03-08 23:04:59 12 NULL 1816-05-05 -1883-04-17 04:14:34.647766229 1883-04-16 02:11:30.647766229 1900s Old Old 1883 2018-03-08 23:04:59 14 NULL 1884-04-16 +1404-07-23 15:32:16.059185026 1404-07-22 13:29:12.059185026 1800s or Earlier Old Old 1404 2018-03-08 23:04:59.000000000 32 NULL 1405-07-23 +1815-05-06 00:12:37.543584705 1815-05-04 22:09:33.543584705 1900s Old Old 1815 2018-03-08 23:04:59.000000000 12 NULL 1816-05-05 +1883-04-17 04:14:34.647766229 1883-04-16 02:11:30.647766229 1900s Old Old 1883 2018-03-08 23:04:59.000000000 14 NULL 1884-04-16 1966-08-16 13:36:50.183618031 1966-08-15 11:33:46.183618031 Early 2010s Old Old 1966 1966-08-16 13:36:50.183618031 36 NULL 1967-08-16 1973-04-17 06:30:38.596784156 1973-04-16 04:27:34.596784156 Early 2010s Old Old 1973 1973-04-17 06:30:38.596784156 30 NULL 1973-04-18 1974-10-04 17:21:03.989 1974-10-03 15:17:59.989 Early 2010s Old Old 1974 1974-10-04 17:21:03.989 21 NULL 1974-10-05 @@ -309,37 +309,37 @@ ctimestamp1 ctimestamp2 ctimestamp2_description ctimestamp2_description_2 ctimes 1987-05-28 13:52:07.900916635 1987-05-27 11:49:03.900916635 Early 2010s Old Old 1987 1987-05-28 13:52:07.900916635 52 NULL 1987-05-29 1998-10-16 20:05:29.397591987 1998-10-15 18:02:25.397591987 Early 2010s Old Old 1998 1998-10-16 20:05:29.397591987 5 NULL 1998-10-17 1999-10-03 16:59:10.396903939 1999-10-02 14:56:06.396903939 Early 2010s Old Old 1999 1999-10-03 16:59:10.396903939 59 NULL 1999-10-04 -2000-12-18 08:42:30.000595596 2000-12-17 06:39:26.000595596 Early 2010s Old Old 2000 2018-03-08 23:04:59 42 NULL 2000-12-19 -2002-05-10 05:29:48.990818073 2002-05-09 03:26:44.990818073 Early 2010s Early 2000s Early 2000s 2002 2018-03-08 23:04:59 29 NULL 2003-05-10 -2003-09-23 22:33:17.00003252 2003-09-22 20:30:13.00003252 Early 2010s Early 2000s Early 2000s 2003 2018-03-08 23:04:59 33 NULL 2003-09-24 -2004-03-07 20:14:13 2004-03-06 18:11:09 Early 2010s Early 2000s Early 2000s 2004 2018-03-08 23:04:59 14 NULL 2004-03-08 -2007-02-09 05:17:29.368756876 2007-02-08 03:14:25.368756876 Late 2000s Late 2000s Late 2000s 2007 2018-03-08 23:04:59 17 NULL 2007-02-10 -2009-01-21 10:49:07.108 2009-01-20 08:46:03.108 Late 2000s Late 2000s Late 2000s 2009 2018-03-08 23:04:59 49 NULL 2009-01-22 -2010-04-08 02:43:35.861742727 2010-04-07 00:40:31.861742727 Late 2000s Late 2000s Late 2000s 2010 2018-03-08 23:04:59 43 NULL 2010-04-09 -2013-04-07 02:44:43.00086821 2013-04-06 00:41:39.00086821 Early 2010s Early 2010s NULL 2013 2018-03-08 23:04:59 44 NULL 2013-04-08 -2013-04-10 00:43:46.854731546 2013-04-08 22:40:42.854731546 Early 2010s Early 2010s NULL 2013 2018-03-08 23:04:59 43 NULL 2013-04-11 -2021-09-24 03:18:32.413655165 2021-09-23 01:15:28.413655165 Unknown NULL NULL 2021 2018-03-08 23:04:59 NULL NULL 2022-09-24 -2024-11-11 16:42:41.101 2024-11-10 14:39:37.101 Unknown NULL NULL 2024 2018-03-08 23:04:59 42 NULL 2024-11-12 -4143-07-08 10:53:27.252802259 4143-07-07 08:50:23.252802259 Unknown NULL NULL 4143 2018-03-08 23:04:59 53 NULL 4143-07-09 -4966-12-04 09:30:55.202 4966-12-03 07:27:51.202 Unknown NULL NULL 4966 2018-03-08 23:04:59 30 NULL 4966-12-05 -5339-02-01 14:10:01.085678691 5339-01-31 12:06:57.085678691 Unknown NULL NULL 5339 2018-03-08 23:04:59 10 NULL 5339-02-02 -5344-10-04 18:40:08.165 5344-10-03 16:37:04.165 Unknown NULL NULL 5344 2018-03-08 23:04:59 40 NULL 5344-10-05 -5397-07-13 07:12:32.000896438 5397-07-12 05:09:28.000896438 Unknown NULL NULL 5397 2018-03-08 23:04:59 12 12 5397-07-14 -5966-07-09 03:30:50.597 5966-07-08 01:27:46.597 Unknown NULL NULL 5966 2018-03-08 23:04:59 30 30 5966-07-10 -6229-06-28 02:54:28.970117179 6229-06-27 00:51:24.970117179 Unknown NULL NULL 6229 2018-03-08 23:04:59 54 54 6229-06-29 -6482-04-27 12:07:38.073915413 6482-04-26 10:04:34.073915413 Unknown NULL NULL 6482 2018-03-08 23:04:59 7 7 6483-04-27 -6631-11-13 16:31:29.702202248 6631-11-12 14:28:25.702202248 Unknown NULL NULL 6631 2018-03-08 23:04:59 31 31 6631-11-14 -6705-09-28 18:27:28.000845672 6705-09-27 16:24:24.000845672 Unknown NULL NULL 6705 2018-03-08 23:04:59 27 NULL 6705-09-29 -6731-02-12 08:12:48.287783702 6731-02-11 06:09:44.287783702 Unknown NULL NULL 6731 2018-03-08 23:04:59 12 NULL 6731-02-13 -7160-12-02 06:00:24.81200852 7160-12-01 03:57:20.81200852 Unknown NULL NULL 7160 2018-03-08 23:04:59 0 NULL 7160-12-03 -7409-09-07 23:33:32.459349602 7409-09-06 21:30:28.459349602 Unknown NULL NULL 7409 2018-03-08 23:04:59 33 NULL 7410-09-07 -7503-06-23 23:14:17.486 7503-06-22 21:11:13.486 Unknown NULL NULL 7503 2018-03-08 23:04:59 14 NULL 7503-06-24 -8422-07-22 03:21:45.745036084 8422-07-21 01:18:41.745036084 Unknown NULL NULL 8422 2018-03-08 23:04:59 21 NULL 8422-07-23 -8521-01-16 20:42:05.668832388 8521-01-15 18:39:01.668832388 Unknown NULL NULL 8521 2018-03-08 23:04:59 42 NULL 8521-01-17 -9075-06-13 16:20:09.218517797 9075-06-12 14:17:05.218517797 Unknown NULL NULL 9075 2018-03-08 23:04:59 20 NULL 9075-06-14 -9209-11-11 04:08:58.223768453 9209-11-10 02:05:54.223768453 Unknown NULL NULL 9209 2018-03-08 23:04:59 8 NULL 9209-11-12 -9403-01-09 18:12:33.547 9403-01-08 16:09:29.547 Unknown NULL NULL 9403 2018-03-08 23:04:59 12 NULL 9404-01-09 -NULL NULL Unknown NULL NULL NULL 2018-03-08 23:04:59 NULL NULL NULL +2000-12-18 08:42:30.000595596 2000-12-17 06:39:26.000595596 Early 2010s Old Old 2000 2018-03-08 23:04:59.000000000 42 NULL 2000-12-19 +2002-05-10 05:29:48.990818073 2002-05-09 03:26:44.990818073 Early 2010s Early 2000s Early 2000s 2002 2018-03-08 23:04:59.000000000 29 NULL 2003-05-10 +2003-09-23 22:33:17.00003252 2003-09-22 20:30:13.00003252 Early 2010s Early 2000s Early 2000s 2003 2018-03-08 23:04:59.000000000 33 NULL 2003-09-24 +2004-03-07 20:14:13 2004-03-06 18:11:09 Early 2010s Early 2000s Early 2000s 2004 2018-03-08 23:04:59.000000000 14 NULL 2004-03-08 +2007-02-09 05:17:29.368756876 2007-02-08 03:14:25.368756876 Late 2000s Late 2000s Late 2000s 2007 2018-03-08 23:04:59.000000000 17 NULL 2007-02-10 +2009-01-21 10:49:07.108 2009-01-20 08:46:03.108 Late 2000s Late 2000s Late 2000s 2009 2018-03-08 23:04:59.000000000 49 NULL 2009-01-22 +2010-04-08 02:43:35.861742727 2010-04-07 00:40:31.861742727 Late 2000s Late 2000s Late 2000s 2010 2018-03-08 23:04:59.000000000 43 NULL 2010-04-09 +2013-04-07 02:44:43.00086821 2013-04-06 00:41:39.00086821 Early 2010s Early 2010s NULL 2013 2018-03-08 23:04:59.000000000 44 NULL 2013-04-08 +2013-04-10 00:43:46.854731546 2013-04-08 22:40:42.854731546 Early 2010s Early 2010s NULL 2013 2018-03-08 23:04:59.000000000 43 NULL 2013-04-11 +2021-09-24 03:18:32.413655165 2021-09-23 01:15:28.413655165 Unknown NULL NULL 2021 2018-03-08 23:04:59.000000000 NULL NULL 2022-09-24 +2024-11-11 16:42:41.101 2024-11-10 14:39:37.101 Unknown NULL NULL 2024 2018-03-08 23:04:59.000000000 42 NULL 2024-11-12 +4143-07-08 10:53:27.252802259 4143-07-07 08:50:23.252802259 Unknown NULL NULL 4143 2018-03-08 23:04:59.000000000 53 NULL 4143-07-09 +4966-12-04 09:30:55.202 4966-12-03 07:27:51.202 Unknown NULL NULL 4966 2018-03-08 23:04:59.000000000 30 NULL 4966-12-05 +5339-02-01 14:10:01.085678691 5339-01-31 12:06:57.085678691 Unknown NULL NULL 5339 2018-03-08 23:04:59.000000000 10 NULL 5339-02-02 +5344-10-04 18:40:08.165 5344-10-03 16:37:04.165 Unknown NULL NULL 5344 2018-03-08 23:04:59.000000000 40 NULL 5344-10-05 +5397-07-13 07:12:32.000896438 5397-07-12 05:09:28.000896438 Unknown NULL NULL 5397 2018-03-08 23:04:59.000000000 12 12 5397-07-14 +5966-07-09 03:30:50.597 5966-07-08 01:27:46.597 Unknown NULL NULL 5966 2018-03-08 23:04:59.000000000 30 30 5966-07-10 +6229-06-28 02:54:28.970117179 6229-06-27 00:51:24.970117179 Unknown NULL NULL 6229 2018-03-08 23:04:59.000000000 54 54 6229-06-29 +6482-04-27 12:07:38.073915413 6482-04-26 10:04:34.073915413 Unknown NULL NULL 6482 2018-03-08 23:04:59.000000000 7 7 6483-04-27 +6631-11-13 16:31:29.702202248 6631-11-12 14:28:25.702202248 Unknown NULL NULL 6631 2018-03-08 23:04:59.000000000 31 31 6631-11-14 +6705-09-28 18:27:28.000845672 6705-09-27 16:24:24.000845672 Unknown NULL NULL 6705 2018-03-08 23:04:59.000000000 27 NULL 6705-09-29 +6731-02-12 08:12:48.287783702 6731-02-11 06:09:44.287783702 Unknown NULL NULL 6731 2018-03-08 23:04:59.000000000 12 NULL 6731-02-13 +7160-12-02 06:00:24.81200852 7160-12-01 03:57:20.81200852 Unknown NULL NULL 7160 2018-03-08 23:04:59.000000000 0 NULL 7160-12-03 +7409-09-07 23:33:32.459349602 7409-09-06 21:30:28.459349602 Unknown NULL NULL 7409 2018-03-08 23:04:59.000000000 33 NULL 7410-09-07 +7503-06-23 23:14:17.486 7503-06-22 21:11:13.486 Unknown NULL NULL 7503 2018-03-08 23:04:59.000000000 14 NULL 7503-06-24 +8422-07-22 03:21:45.745036084 8422-07-21 01:18:41.745036084 Unknown NULL NULL 8422 2018-03-08 23:04:59.000000000 21 NULL 8422-07-23 +8521-01-16 20:42:05.668832388 8521-01-15 18:39:01.668832388 Unknown NULL NULL 8521 2018-03-08 23:04:59.000000000 42 NULL 8521-01-17 +9075-06-13 16:20:09.218517797 9075-06-12 14:17:05.218517797 Unknown NULL NULL 9075 2018-03-08 23:04:59.000000000 20 NULL 9075-06-14 +9209-11-11 04:08:58.223768453 9209-11-10 02:05:54.223768453 Unknown NULL NULL 9209 2018-03-08 23:04:59.000000000 8 NULL 9209-11-12 +9403-01-09 18:12:33.547 9403-01-08 16:09:29.547 Unknown NULL NULL 9403 2018-03-08 23:04:59.000000000 12 NULL 9404-01-09 +NULL NULL Unknown NULL NULL NULL 2018-03-08 23:04:59.000000000 NULL NULL NULL PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT ctimestamp1, @@ -434,14 +434,14 @@ STAGE PLANS: native: true vectorizationSchemaColumns: [0:cdate:date, 1:ctimestamp1:timestamp, 2:stimestamp1:string, 3:ctimestamp2:timestamp, 4:ROW__ID:struct, 5:ROW__IS__DELETED:boolean] Select Operator - expressions: ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), CASE WHEN ((ctimestamp2 <= TIMESTAMP'1800-12-31 00:00:00')) THEN ('1800s or Earlier') WHEN ((ctimestamp2 < TIMESTAMP'1900-01-01 00:00:00')) THEN ('1900s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN ('Early 2010s') ELSE ('Unknown') END (type: string), CASE WHEN ((ctimestamp2 <= TIMESTAMP'2000-12-31 23:59:59.999999999')) THEN ('Old') WHEN ((ctimestamp2 < TIMESTAMP'2006-01-01 00:00:00')) THEN ('Early 2000s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN ('Early 2010s') ELSE (null) END (type: string), CASE WHEN ((ctimestamp2 <= TIMESTAMP'2000-12-31 23:59:59.999999999')) THEN ('Old') WHEN ((ctimestamp2 < TIMESTAMP'2006-01-01 00:00:00')) THEN ('Early 2000s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') ELSE (null) END (type: string), if((ctimestamp1 < TIMESTAMP'1974-10-04 17:21:03.989'), year(ctimestamp1), year(ctimestamp2)) (type: int), if((stimestamp1 like '%19%'), stimestamp1, '2018-03-08 23:04:59') (type: string), if((ctimestamp1 = TIMESTAMP'2021-09-24 03:18:32.413655165'), null, minute(ctimestamp1)) (type: int), if(((ctimestamp2 >= TIMESTAMP'5344-10-04 18:40:08.165') and (ctimestamp2 < TIMESTAMP'6631-11-13 16:31:29.702202248')), minute(ctimestamp1), null) (type: int), if(((UDFToDouble(ctimestamp1) % 500.0D) > 100.0D), date_add(cdate, 1), date_add(cdate, 365)) (type: date), stimestamp1 (type: string) + expressions: ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), CASE WHEN ((ctimestamp2 <= TIMESTAMP'1800-12-31 00:00:00')) THEN ('1800s or Earlier') WHEN ((ctimestamp2 < TIMESTAMP'1900-01-01 00:00:00')) THEN ('1900s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN ('Early 2010s') ELSE ('Unknown') END (type: string), CASE WHEN ((ctimestamp2 <= TIMESTAMP'2000-12-31 23:59:59.999999999')) THEN ('Old') WHEN ((ctimestamp2 < TIMESTAMP'2006-01-01 00:00:00')) THEN ('Early 2000s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN ('Early 2010s') ELSE (null) END (type: string), CASE WHEN ((ctimestamp2 <= TIMESTAMP'2000-12-31 23:59:59.999999999')) THEN ('Old') WHEN ((ctimestamp2 < TIMESTAMP'2006-01-01 00:00:00')) THEN ('Early 2000s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') ELSE (null) END (type: string), if((ctimestamp1 < TIMESTAMP'1974-10-04 17:21:03.989'), year(ctimestamp1), year(ctimestamp2)) (type: int), if((stimestamp1 like '%19%'), stimestamp1, '2018-03-08 23:04:59.000000000') (type: string), if((ctimestamp1 = TIMESTAMP'2021-09-24 03:18:32.413655165'), null, minute(ctimestamp1)) (type: int), if(((ctimestamp2 >= TIMESTAMP'5344-10-04 18:40:08.165') and (ctimestamp2 < TIMESTAMP'6631-11-13 16:31:29.702202248')), minute(ctimestamp1), null) (type: int), if(((UDFToDouble(ctimestamp1) % 500.0D) > 100.0D), date_add(cdate, 1), date_add(cdate, 365)) (type: date), stimestamp1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 Select Vectorization: className: VectorSelectOperator native: true projectedOutputColumnNums: [1, 3, 13, 22, 29, 33, 35, 38, 43, 49, 2] - selectExpressions: IfExprStringScalarStringGroupColumn(col 6:boolean, val 1800s or Earliercol 12:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 1800-12-31 00:00:00) -> 6:boolean, IfExprStringScalarStringGroupColumn(col 7:boolean, val 1900scol 11:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 1900-01-01 00:00:00) -> 7:boolean, IfExprStringScalarStringGroupColumn(col 8:boolean, val Late 2000scol 10:string)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 8:boolean, IfExprStringScalarStringScalar(col 9:boolean, val Early 2010s, val Unknown)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 9:boolean) -> 10:string) -> 11:string) -> 12:string) -> 13:string, IfExprStringScalarStringGroupColumn(col 14:boolean, val Oldcol 21:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2000-12-31 23:59:59.999999999) -> 14:boolean, IfExprStringScalarStringGroupColumn(col 15:boolean, val Early 2000scol 20:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 15:boolean, IfExprStringScalarStringGroupColumn(col 16:boolean, val Late 2000scol 19:string)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 16:boolean, IfExprColumnNull(col 17:boolean, col 18:string, null)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 17:boolean, ConstantVectorExpression(val Early 2010s) -> 18:string) -> 19:string) -> 20:string) -> 21:string) -> 22:string, IfExprStringScalarStringGroupColumn(col 23:boolean, val Oldcol 28:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2000-12-31 23:59:59.999999999) -> 23:boolean, IfExprStringScalarStringGroupColumn(col 24:boolean, val Early 2000scol 27:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 24:boolean, IfExprColumnNull(col 25:boolean, col 26:string, null)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 25:boolean, ConstantVectorExpression(val Late 2000s) -> 26:string) -> 27:string) -> 28:string) -> 29:string, IfExprLongColumnLongColumn(col 30:boolean, col 31:int, col 32:int)(children: TimestampColLessTimestampScalar(col 1:timestamp, val 1974-10-04 17:21:03.989) -> 30:boolean, VectorUDFYearTimestamp(col 1:timestamp, field YEAR) -> 31:int, VectorUDFYearTimestamp(col 3:timestamp, field YEAR) -> 32:int) -> 33:int, IfExprStringGroupColumnStringScalar(col 34:boolean, col 2:string, val 2018-03-08 23:04:59)(children: SelectStringColLikeStringScalar(col 2:string) -> 34:boolean) -> 35:string, IfExprNullColumn(col 36:boolean, null, col 37)(children: TimestampColEqualTimestampScalar(col 1:timestamp, val 2021-09-24 03:18:32.413655165) -> 36:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 37:int) -> 38:int, IfExprColumnNull(col 41:boolean, col 42:int, null)(children: ColAndCol(col 39:boolean, col 40:boolean)(children: TimestampColGreaterEqualTimestampScalar(col 3:timestamp, val 5344-10-04 18:40:08.165) -> 39:boolean, TimestampColLessTimestampScalar(col 3:timestamp, val 6631-11-13 16:31:29.702202248) -> 40:boolean) -> 41:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 42:int) -> 43:int, IfExprLongColumnLongColumn(col 46:boolean, col 47:date, col 48:date)(children: DoubleColGreaterDoubleScalar(col 45:double, val 100.0)(children: DoubleColModuloDoubleScalar(col 44:double, val 500.0)(children: CastTimestampToDouble(col 1:timestamp) -> 44:double) -> 45:double) -> 46:boolean, VectorUDFDateAddColScalar(col 0:date, val 1) -> 47:date, VectorUDFDateAddColScalar(col 0:date, val 365) -> 48:date) -> 49:date - Statistics: Num rows: 51 Data size: 23760 Basic stats: COMPLETE Column stats: COMPLETE + selectExpressions: IfExprStringScalarStringGroupColumn(col 6:boolean, val 1800s or Earliercol 12:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 1800-12-31 00:00:00) -> 6:boolean, IfExprStringScalarStringGroupColumn(col 7:boolean, val 1900scol 11:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 1900-01-01 00:00:00) -> 7:boolean, IfExprStringScalarStringGroupColumn(col 8:boolean, val Late 2000scol 10:string)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 8:boolean, IfExprStringScalarStringScalar(col 9:boolean, val Early 2010s, val Unknown)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 9:boolean) -> 10:string) -> 11:string) -> 12:string) -> 13:string, IfExprStringScalarStringGroupColumn(col 14:boolean, val Oldcol 21:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2000-12-31 23:59:59.999999999) -> 14:boolean, IfExprStringScalarStringGroupColumn(col 15:boolean, val Early 2000scol 20:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 15:boolean, IfExprStringScalarStringGroupColumn(col 16:boolean, val Late 2000scol 19:string)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 16:boolean, IfExprColumnNull(col 17:boolean, col 18:string, null)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 17:boolean, ConstantVectorExpression(val Early 2010s) -> 18:string) -> 19:string) -> 20:string) -> 21:string) -> 22:string, IfExprStringScalarStringGroupColumn(col 23:boolean, val Oldcol 28:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2000-12-31 23:59:59.999999999) -> 23:boolean, IfExprStringScalarStringGroupColumn(col 24:boolean, val Early 2000scol 27:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 24:boolean, IfExprColumnNull(col 25:boolean, col 26:string, null)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 25:boolean, ConstantVectorExpression(val Late 2000s) -> 26:string) -> 27:string) -> 28:string) -> 29:string, IfExprLongColumnLongColumn(col 30:boolean, col 31:int, col 32:int)(children: TimestampColLessTimestampScalar(col 1:timestamp, val 1974-10-04 17:21:03.989) -> 30:boolean, VectorUDFYearTimestamp(col 1:timestamp, field YEAR) -> 31:int, VectorUDFYearTimestamp(col 3:timestamp, field YEAR) -> 32:int) -> 33:int, IfExprStringGroupColumnStringScalar(col 34:boolean, col 2:string, val 2018-03-08 23:04:59.000000000)(children: SelectStringColLikeStringScalar(col 2:string) -> 34:boolean) -> 35:string, IfExprNullColumn(col 36:boolean, null, col 37)(children: TimestampColEqualTimestampScalar(col 1:timestamp, val 2021-09-24 03:18:32.413655165) -> 36:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 37:int) -> 38:int, IfExprColumnNull(col 41:boolean, col 42:int, null)(children: ColAndCol(col 39:boolean, col 40:boolean)(children: TimestampColGreaterEqualTimestampScalar(col 3:timestamp, val 5344-10-04 18:40:08.165) -> 39:boolean, TimestampColLessTimestampScalar(col 3:timestamp, val 6631-11-13 16:31:29.702202248) -> 40:boolean) -> 41:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 42:int) -> 43:int, IfExprLongColumnLongColumn(col 46:boolean, col 47:date, col 48:date)(children: DoubleColGreaterDoubleScalar(col 45:double, val 100.0)(children: DoubleColModuloDoubleScalar(col 44:double, val 500.0)(children: CastTimestampToDouble(col 1:timestamp) -> 44:double) -> 45:double) -> 46:boolean, VectorUDFDateAddColScalar(col 0:date, val 1) -> 47:date, VectorUDFDateAddColScalar(col 0:date, val 365) -> 48:date) -> 49:date + Statistics: Num rows: 51 Data size: 23862 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: timestamp), _col10 (type: string), _col1 (type: timestamp) null sort order: zzz @@ -452,7 +452,7 @@ STAGE PLANS: native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true valueColumns: 13:string, 22:string, 29:string, 33:int, 35:string, 38:int, 43:int, 49:date - Statistics: Num rows: 51 Data size: 23760 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 51 Data size: 23862 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int), _col8 (type: int), _col9 (type: date) Execution mode: vectorized, llap LLAP IO: all inputs @@ -494,13 +494,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 2, 3, 4, 5, 6, 7, 8, 9, 10] - Statistics: Num rows: 51 Data size: 18099 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 51 Data size: 18201 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 51 Data size: 18099 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 51 Data size: 18201 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -580,11 +580,11 @@ POSTHOOK: Input: default@timestamps #### A masked pattern was here #### ctimestamp1 ctimestamp2 ctimestamp2_description ctimestamp2_description_2 ctimestamp2_description_3 field1 field_2 field_3 field_4 field_5 0004-09-22 18:26:29.519542222 0004-09-21 16:23:25.519542222 1800s or Earlier Old Old 4 0004-09-22 18:26:29.519542222 26 NULL 0005-09-22 -0528-10-27 08:15:18.941718273 0528-10-26 06:12:14.941718273 1800s or Earlier Old Old 528 2018-03-08 23:04:59 15 NULL 0529-10-27 +0528-10-27 08:15:18.941718273 0528-10-26 06:12:14.941718273 1800s or Earlier Old Old 528 2018-03-08 23:04:59.000000000 15 NULL 0529-10-27 1319-02-02 16:31:57.778 1319-02-01 14:28:53.778 1800s or Earlier Old Old 1319 1319-02-02 16:31:57.778 31 NULL 1320-02-02 -1404-07-23 15:32:16.059185026 1404-07-22 13:29:12.059185026 1800s or Earlier Old Old 1404 2018-03-08 23:04:59 32 NULL 1405-07-23 -1815-05-06 00:12:37.543584705 1815-05-04 22:09:33.543584705 1900s Old Old 1815 2018-03-08 23:04:59 12 NULL 1816-05-05 -1883-04-17 04:14:34.647766229 1883-04-16 02:11:30.647766229 1900s Old Old 1883 2018-03-08 23:04:59 14 NULL 1884-04-16 +1404-07-23 15:32:16.059185026 1404-07-22 13:29:12.059185026 1800s or Earlier Old Old 1404 2018-03-08 23:04:59.000000000 32 NULL 1405-07-23 +1815-05-06 00:12:37.543584705 1815-05-04 22:09:33.543584705 1900s Old Old 1815 2018-03-08 23:04:59.000000000 12 NULL 1816-05-05 +1883-04-17 04:14:34.647766229 1883-04-16 02:11:30.647766229 1900s Old Old 1883 2018-03-08 23:04:59.000000000 14 NULL 1884-04-16 1966-08-16 13:36:50.183618031 1966-08-15 11:33:46.183618031 Early 2010s Old Old 1966 1966-08-16 13:36:50.183618031 36 NULL 1967-08-16 1973-04-17 06:30:38.596784156 1973-04-16 04:27:34.596784156 Early 2010s Old Old 1973 1973-04-17 06:30:38.596784156 30 NULL 1973-04-18 1974-10-04 17:21:03.989 1974-10-03 15:17:59.989 Early 2010s Old Old 1974 1974-10-04 17:21:03.989 21 NULL 1974-10-05 @@ -599,37 +599,37 @@ ctimestamp1 ctimestamp2 ctimestamp2_description ctimestamp2_description_2 ctimes 1987-05-28 13:52:07.900916635 1987-05-27 11:49:03.900916635 Early 2010s Old Old 1987 1987-05-28 13:52:07.900916635 52 NULL 1987-05-29 1998-10-16 20:05:29.397591987 1998-10-15 18:02:25.397591987 Early 2010s Old Old 1998 1998-10-16 20:05:29.397591987 5 NULL 1998-10-17 1999-10-03 16:59:10.396903939 1999-10-02 14:56:06.396903939 Early 2010s Old Old 1999 1999-10-03 16:59:10.396903939 59 NULL 1999-10-04 -2000-12-18 08:42:30.000595596 2000-12-17 06:39:26.000595596 Early 2010s Old Old 2000 2018-03-08 23:04:59 42 NULL 2000-12-19 -2002-05-10 05:29:48.990818073 2002-05-09 03:26:44.990818073 Early 2010s Early 2000s Early 2000s 2002 2018-03-08 23:04:59 29 NULL 2003-05-10 -2003-09-23 22:33:17.00003252 2003-09-22 20:30:13.00003252 Early 2010s Early 2000s Early 2000s 2003 2018-03-08 23:04:59 33 NULL 2003-09-24 -2004-03-07 20:14:13 2004-03-06 18:11:09 Early 2010s Early 2000s Early 2000s 2004 2018-03-08 23:04:59 14 NULL 2004-03-08 -2007-02-09 05:17:29.368756876 2007-02-08 03:14:25.368756876 Late 2000s Late 2000s Late 2000s 2007 2018-03-08 23:04:59 17 NULL 2007-02-10 -2009-01-21 10:49:07.108 2009-01-20 08:46:03.108 Late 2000s Late 2000s Late 2000s 2009 2018-03-08 23:04:59 49 NULL 2009-01-22 -2010-04-08 02:43:35.861742727 2010-04-07 00:40:31.861742727 Late 2000s Late 2000s Late 2000s 2010 2018-03-08 23:04:59 43 NULL 2010-04-09 -2013-04-07 02:44:43.00086821 2013-04-06 00:41:39.00086821 Early 2010s Early 2010s NULL 2013 2018-03-08 23:04:59 44 NULL 2013-04-08 -2013-04-10 00:43:46.854731546 2013-04-08 22:40:42.854731546 Early 2010s Early 2010s NULL 2013 2018-03-08 23:04:59 43 NULL 2013-04-11 -2021-09-24 03:18:32.413655165 2021-09-23 01:15:28.413655165 Unknown NULL NULL 2021 2018-03-08 23:04:59 NULL NULL 2022-09-24 -2024-11-11 16:42:41.101 2024-11-10 14:39:37.101 Unknown NULL NULL 2024 2018-03-08 23:04:59 42 NULL 2024-11-12 -4143-07-08 10:53:27.252802259 4143-07-07 08:50:23.252802259 Unknown NULL NULL 4143 2018-03-08 23:04:59 53 NULL 4143-07-09 -4966-12-04 09:30:55.202 4966-12-03 07:27:51.202 Unknown NULL NULL 4966 2018-03-08 23:04:59 30 NULL 4966-12-05 -5339-02-01 14:10:01.085678691 5339-01-31 12:06:57.085678691 Unknown NULL NULL 5339 2018-03-08 23:04:59 10 NULL 5339-02-02 -5344-10-04 18:40:08.165 5344-10-03 16:37:04.165 Unknown NULL NULL 5344 2018-03-08 23:04:59 40 NULL 5344-10-05 -5397-07-13 07:12:32.000896438 5397-07-12 05:09:28.000896438 Unknown NULL NULL 5397 2018-03-08 23:04:59 12 12 5397-07-14 -5966-07-09 03:30:50.597 5966-07-08 01:27:46.597 Unknown NULL NULL 5966 2018-03-08 23:04:59 30 30 5966-07-10 -6229-06-28 02:54:28.970117179 6229-06-27 00:51:24.970117179 Unknown NULL NULL 6229 2018-03-08 23:04:59 54 54 6229-06-29 -6482-04-27 12:07:38.073915413 6482-04-26 10:04:34.073915413 Unknown NULL NULL 6482 2018-03-08 23:04:59 7 7 6483-04-27 -6631-11-13 16:31:29.702202248 6631-11-12 14:28:25.702202248 Unknown NULL NULL 6631 2018-03-08 23:04:59 31 31 6631-11-14 -6705-09-28 18:27:28.000845672 6705-09-27 16:24:24.000845672 Unknown NULL NULL 6705 2018-03-08 23:04:59 27 NULL 6705-09-29 -6731-02-12 08:12:48.287783702 6731-02-11 06:09:44.287783702 Unknown NULL NULL 6731 2018-03-08 23:04:59 12 NULL 6731-02-13 -7160-12-02 06:00:24.81200852 7160-12-01 03:57:20.81200852 Unknown NULL NULL 7160 2018-03-08 23:04:59 0 NULL 7160-12-03 -7409-09-07 23:33:32.459349602 7409-09-06 21:30:28.459349602 Unknown NULL NULL 7409 2018-03-08 23:04:59 33 NULL 7410-09-07 -7503-06-23 23:14:17.486 7503-06-22 21:11:13.486 Unknown NULL NULL 7503 2018-03-08 23:04:59 14 NULL 7503-06-24 -8422-07-22 03:21:45.745036084 8422-07-21 01:18:41.745036084 Unknown NULL NULL 8422 2018-03-08 23:04:59 21 NULL 8422-07-23 -8521-01-16 20:42:05.668832388 8521-01-15 18:39:01.668832388 Unknown NULL NULL 8521 2018-03-08 23:04:59 42 NULL 8521-01-17 -9075-06-13 16:20:09.218517797 9075-06-12 14:17:05.218517797 Unknown NULL NULL 9075 2018-03-08 23:04:59 20 NULL 9075-06-14 -9209-11-11 04:08:58.223768453 9209-11-10 02:05:54.223768453 Unknown NULL NULL 9209 2018-03-08 23:04:59 8 NULL 9209-11-12 -9403-01-09 18:12:33.547 9403-01-08 16:09:29.547 Unknown NULL NULL 9403 2018-03-08 23:04:59 12 NULL 9404-01-09 -NULL NULL Unknown NULL NULL NULL 2018-03-08 23:04:59 NULL NULL NULL +2000-12-18 08:42:30.000595596 2000-12-17 06:39:26.000595596 Early 2010s Old Old 2000 2018-03-08 23:04:59.000000000 42 NULL 2000-12-19 +2002-05-10 05:29:48.990818073 2002-05-09 03:26:44.990818073 Early 2010s Early 2000s Early 2000s 2002 2018-03-08 23:04:59.000000000 29 NULL 2003-05-10 +2003-09-23 22:33:17.00003252 2003-09-22 20:30:13.00003252 Early 2010s Early 2000s Early 2000s 2003 2018-03-08 23:04:59.000000000 33 NULL 2003-09-24 +2004-03-07 20:14:13 2004-03-06 18:11:09 Early 2010s Early 2000s Early 2000s 2004 2018-03-08 23:04:59.000000000 14 NULL 2004-03-08 +2007-02-09 05:17:29.368756876 2007-02-08 03:14:25.368756876 Late 2000s Late 2000s Late 2000s 2007 2018-03-08 23:04:59.000000000 17 NULL 2007-02-10 +2009-01-21 10:49:07.108 2009-01-20 08:46:03.108 Late 2000s Late 2000s Late 2000s 2009 2018-03-08 23:04:59.000000000 49 NULL 2009-01-22 +2010-04-08 02:43:35.861742727 2010-04-07 00:40:31.861742727 Late 2000s Late 2000s Late 2000s 2010 2018-03-08 23:04:59.000000000 43 NULL 2010-04-09 +2013-04-07 02:44:43.00086821 2013-04-06 00:41:39.00086821 Early 2010s Early 2010s NULL 2013 2018-03-08 23:04:59.000000000 44 NULL 2013-04-08 +2013-04-10 00:43:46.854731546 2013-04-08 22:40:42.854731546 Early 2010s Early 2010s NULL 2013 2018-03-08 23:04:59.000000000 43 NULL 2013-04-11 +2021-09-24 03:18:32.413655165 2021-09-23 01:15:28.413655165 Unknown NULL NULL 2021 2018-03-08 23:04:59.000000000 NULL NULL 2022-09-24 +2024-11-11 16:42:41.101 2024-11-10 14:39:37.101 Unknown NULL NULL 2024 2018-03-08 23:04:59.000000000 42 NULL 2024-11-12 +4143-07-08 10:53:27.252802259 4143-07-07 08:50:23.252802259 Unknown NULL NULL 4143 2018-03-08 23:04:59.000000000 53 NULL 4143-07-09 +4966-12-04 09:30:55.202 4966-12-03 07:27:51.202 Unknown NULL NULL 4966 2018-03-08 23:04:59.000000000 30 NULL 4966-12-05 +5339-02-01 14:10:01.085678691 5339-01-31 12:06:57.085678691 Unknown NULL NULL 5339 2018-03-08 23:04:59.000000000 10 NULL 5339-02-02 +5344-10-04 18:40:08.165 5344-10-03 16:37:04.165 Unknown NULL NULL 5344 2018-03-08 23:04:59.000000000 40 NULL 5344-10-05 +5397-07-13 07:12:32.000896438 5397-07-12 05:09:28.000896438 Unknown NULL NULL 5397 2018-03-08 23:04:59.000000000 12 12 5397-07-14 +5966-07-09 03:30:50.597 5966-07-08 01:27:46.597 Unknown NULL NULL 5966 2018-03-08 23:04:59.000000000 30 30 5966-07-10 +6229-06-28 02:54:28.970117179 6229-06-27 00:51:24.970117179 Unknown NULL NULL 6229 2018-03-08 23:04:59.000000000 54 54 6229-06-29 +6482-04-27 12:07:38.073915413 6482-04-26 10:04:34.073915413 Unknown NULL NULL 6482 2018-03-08 23:04:59.000000000 7 7 6483-04-27 +6631-11-13 16:31:29.702202248 6631-11-12 14:28:25.702202248 Unknown NULL NULL 6631 2018-03-08 23:04:59.000000000 31 31 6631-11-14 +6705-09-28 18:27:28.000845672 6705-09-27 16:24:24.000845672 Unknown NULL NULL 6705 2018-03-08 23:04:59.000000000 27 NULL 6705-09-29 +6731-02-12 08:12:48.287783702 6731-02-11 06:09:44.287783702 Unknown NULL NULL 6731 2018-03-08 23:04:59.000000000 12 NULL 6731-02-13 +7160-12-02 06:00:24.81200852 7160-12-01 03:57:20.81200852 Unknown NULL NULL 7160 2018-03-08 23:04:59.000000000 0 NULL 7160-12-03 +7409-09-07 23:33:32.459349602 7409-09-06 21:30:28.459349602 Unknown NULL NULL 7409 2018-03-08 23:04:59.000000000 33 NULL 7410-09-07 +7503-06-23 23:14:17.486 7503-06-22 21:11:13.486 Unknown NULL NULL 7503 2018-03-08 23:04:59.000000000 14 NULL 7503-06-24 +8422-07-22 03:21:45.745036084 8422-07-21 01:18:41.745036084 Unknown NULL NULL 8422 2018-03-08 23:04:59.000000000 21 NULL 8422-07-23 +8521-01-16 20:42:05.668832388 8521-01-15 18:39:01.668832388 Unknown NULL NULL 8521 2018-03-08 23:04:59.000000000 42 NULL 8521-01-17 +9075-06-13 16:20:09.218517797 9075-06-12 14:17:05.218517797 Unknown NULL NULL 9075 2018-03-08 23:04:59.000000000 20 NULL 9075-06-14 +9209-11-11 04:08:58.223768453 9209-11-10 02:05:54.223768453 Unknown NULL NULL 9209 2018-03-08 23:04:59.000000000 8 NULL 9209-11-12 +9403-01-09 18:12:33.547 9403-01-08 16:09:29.547 Unknown NULL NULL 9403 2018-03-08 23:04:59.000000000 12 NULL 9404-01-09 +NULL NULL Unknown NULL NULL NULL 2018-03-08 23:04:59.000000000 NULL NULL NULL PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT ctimestamp1, @@ -724,14 +724,14 @@ STAGE PLANS: native: true vectorizationSchemaColumns: [0:cdate:date, 1:ctimestamp1:timestamp, 2:stimestamp1:string, 3:ctimestamp2:timestamp, 4:ROW__ID:struct, 5:ROW__IS__DELETED:boolean] Select Operator - expressions: ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), CASE WHEN ((ctimestamp2 <= TIMESTAMP'1800-12-31 00:00:00')) THEN ('1800s or Earlier') WHEN ((ctimestamp2 < TIMESTAMP'1900-01-01 00:00:00')) THEN ('1900s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN ('Early 2010s') ELSE ('Unknown') END (type: string), CASE WHEN ((ctimestamp2 <= TIMESTAMP'2000-12-31 23:59:59.999999999')) THEN ('Old') WHEN ((ctimestamp2 < TIMESTAMP'2006-01-01 00:00:00')) THEN ('Early 2000s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN ('Early 2010s') ELSE (null) END (type: string), CASE WHEN ((ctimestamp2 <= TIMESTAMP'2000-12-31 23:59:59.999999999')) THEN ('Old') WHEN ((ctimestamp2 < TIMESTAMP'2006-01-01 00:00:00')) THEN ('Early 2000s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') ELSE (null) END (type: string), if((ctimestamp1 < TIMESTAMP'1974-10-04 17:21:03.989'), year(ctimestamp1), year(ctimestamp2)) (type: int), if((stimestamp1 like '%19%'), stimestamp1, '2018-03-08 23:04:59') (type: string), if((ctimestamp1 = TIMESTAMP'2021-09-24 03:18:32.413655165'), null, minute(ctimestamp1)) (type: int), if(((ctimestamp2 >= TIMESTAMP'5344-10-04 18:40:08.165') and (ctimestamp2 < TIMESTAMP'6631-11-13 16:31:29.702202248')), minute(ctimestamp1), null) (type: int), if(((UDFToDouble(ctimestamp1) % 500.0D) > 100.0D), date_add(cdate, 1), date_add(cdate, 365)) (type: date), stimestamp1 (type: string) + expressions: ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), CASE WHEN ((ctimestamp2 <= TIMESTAMP'1800-12-31 00:00:00')) THEN ('1800s or Earlier') WHEN ((ctimestamp2 < TIMESTAMP'1900-01-01 00:00:00')) THEN ('1900s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN ('Early 2010s') ELSE ('Unknown') END (type: string), CASE WHEN ((ctimestamp2 <= TIMESTAMP'2000-12-31 23:59:59.999999999')) THEN ('Old') WHEN ((ctimestamp2 < TIMESTAMP'2006-01-01 00:00:00')) THEN ('Early 2000s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN ('Early 2010s') ELSE (null) END (type: string), CASE WHEN ((ctimestamp2 <= TIMESTAMP'2000-12-31 23:59:59.999999999')) THEN ('Old') WHEN ((ctimestamp2 < TIMESTAMP'2006-01-01 00:00:00')) THEN ('Early 2000s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') ELSE (null) END (type: string), if((ctimestamp1 < TIMESTAMP'1974-10-04 17:21:03.989'), year(ctimestamp1), year(ctimestamp2)) (type: int), if((stimestamp1 like '%19%'), stimestamp1, '2018-03-08 23:04:59.000000000') (type: string), if((ctimestamp1 = TIMESTAMP'2021-09-24 03:18:32.413655165'), null, minute(ctimestamp1)) (type: int), if(((ctimestamp2 >= TIMESTAMP'5344-10-04 18:40:08.165') and (ctimestamp2 < TIMESTAMP'6631-11-13 16:31:29.702202248')), minute(ctimestamp1), null) (type: int), if(((UDFToDouble(ctimestamp1) % 500.0D) > 100.0D), date_add(cdate, 1), date_add(cdate, 365)) (type: date), stimestamp1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 Select Vectorization: className: VectorSelectOperator native: true projectedOutputColumnNums: [1, 3, 16, 28, 37, 41, 43, 46, 51, 57, 2] - selectExpressions: IfExprColumnCondExpr(col 6:boolean, col 7:stringcol 15:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 1800-12-31 00:00:00) -> 6:boolean, ConstantVectorExpression(val 1800s or Earlier) -> 7:string, IfExprColumnCondExpr(col 8:boolean, col 9:stringcol 14:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 1900-01-01 00:00:00) -> 8:boolean, ConstantVectorExpression(val 1900s) -> 9:string, IfExprColumnCondExpr(col 10:boolean, col 11:stringcol 13:string)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 10:boolean, ConstantVectorExpression(val Late 2000s) -> 11:string, IfExprStringScalarStringScalar(col 12:boolean, val Early 2010s, val Unknown)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 12:boolean) -> 13:string) -> 14:string) -> 15:string) -> 16:string, IfExprColumnCondExpr(col 17:boolean, col 18:stringcol 27:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2000-12-31 23:59:59.999999999) -> 17:boolean, ConstantVectorExpression(val Old) -> 18:string, IfExprColumnCondExpr(col 19:boolean, col 20:stringcol 26:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 19:boolean, ConstantVectorExpression(val Early 2000s) -> 20:string, IfExprColumnCondExpr(col 21:boolean, col 22:stringcol 25:string)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 21:boolean, ConstantVectorExpression(val Late 2000s) -> 22:string, IfExprColumnNull(col 23:boolean, col 24:string, null)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 23:boolean, ConstantVectorExpression(val Early 2010s) -> 24:string) -> 25:string) -> 26:string) -> 27:string) -> 28:string, IfExprColumnCondExpr(col 29:boolean, col 30:stringcol 36:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2000-12-31 23:59:59.999999999) -> 29:boolean, ConstantVectorExpression(val Old) -> 30:string, IfExprColumnCondExpr(col 31:boolean, col 32:stringcol 35:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 31:boolean, ConstantVectorExpression(val Early 2000s) -> 32:string, IfExprColumnNull(col 33:boolean, col 34:string, null)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 33:boolean, ConstantVectorExpression(val Late 2000s) -> 34:string) -> 35:string) -> 36:string) -> 37:string, IfExprCondExprCondExpr(col 38:boolean, col 39:int, col 40:int)(children: TimestampColLessTimestampScalar(col 1:timestamp, val 1974-10-04 17:21:03.989) -> 38:boolean, VectorUDFYearTimestamp(col 1:timestamp, field YEAR) -> 39:int, VectorUDFYearTimestamp(col 3:timestamp, field YEAR) -> 40:int) -> 41:int, IfExprStringGroupColumnStringScalar(col 42:boolean, col 2:string, val 2018-03-08 23:04:59)(children: SelectStringColLikeStringScalar(col 2:string) -> 42:boolean) -> 43:string, IfExprNullCondExpr(col 44:boolean, null, col 45:int)(children: TimestampColEqualTimestampScalar(col 1:timestamp, val 2021-09-24 03:18:32.413655165) -> 44:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 45:int) -> 46:int, IfExprCondExprNull(col 49:boolean, col 50:int, null)(children: ColAndCol(col 47:boolean, col 48:boolean)(children: TimestampColGreaterEqualTimestampScalar(col 3:timestamp, val 5344-10-04 18:40:08.165) -> 47:boolean, TimestampColLessTimestampScalar(col 3:timestamp, val 6631-11-13 16:31:29.702202248) -> 48:boolean) -> 49:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 50:int) -> 51:int, IfExprCondExprCondExpr(col 54:boolean, col 55:date, col 56:date)(children: DoubleColGreaterDoubleScalar(col 53:double, val 100.0)(children: DoubleColModuloDoubleScalar(col 52:double, val 500.0)(children: CastTimestampToDouble(col 1:timestamp) -> 52:double) -> 53:double) -> 54:boolean, VectorUDFDateAddColScalar(col 0:date, val 1) -> 55:date, VectorUDFDateAddColScalar(col 0:date, val 365) -> 56:date) -> 57:date - Statistics: Num rows: 51 Data size: 23760 Basic stats: COMPLETE Column stats: COMPLETE + selectExpressions: IfExprColumnCondExpr(col 6:boolean, col 7:stringcol 15:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 1800-12-31 00:00:00) -> 6:boolean, ConstantVectorExpression(val 1800s or Earlier) -> 7:string, IfExprColumnCondExpr(col 8:boolean, col 9:stringcol 14:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 1900-01-01 00:00:00) -> 8:boolean, ConstantVectorExpression(val 1900s) -> 9:string, IfExprColumnCondExpr(col 10:boolean, col 11:stringcol 13:string)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 10:boolean, ConstantVectorExpression(val Late 2000s) -> 11:string, IfExprStringScalarStringScalar(col 12:boolean, val Early 2010s, val Unknown)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 12:boolean) -> 13:string) -> 14:string) -> 15:string) -> 16:string, IfExprColumnCondExpr(col 17:boolean, col 18:stringcol 27:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2000-12-31 23:59:59.999999999) -> 17:boolean, ConstantVectorExpression(val Old) -> 18:string, IfExprColumnCondExpr(col 19:boolean, col 20:stringcol 26:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 19:boolean, ConstantVectorExpression(val Early 2000s) -> 20:string, IfExprColumnCondExpr(col 21:boolean, col 22:stringcol 25:string)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 21:boolean, ConstantVectorExpression(val Late 2000s) -> 22:string, IfExprColumnNull(col 23:boolean, col 24:string, null)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 23:boolean, ConstantVectorExpression(val Early 2010s) -> 24:string) -> 25:string) -> 26:string) -> 27:string) -> 28:string, IfExprColumnCondExpr(col 29:boolean, col 30:stringcol 36:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2000-12-31 23:59:59.999999999) -> 29:boolean, ConstantVectorExpression(val Old) -> 30:string, IfExprColumnCondExpr(col 31:boolean, col 32:stringcol 35:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 31:boolean, ConstantVectorExpression(val Early 2000s) -> 32:string, IfExprColumnNull(col 33:boolean, col 34:string, null)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 33:boolean, ConstantVectorExpression(val Late 2000s) -> 34:string) -> 35:string) -> 36:string) -> 37:string, IfExprCondExprCondExpr(col 38:boolean, col 39:int, col 40:int)(children: TimestampColLessTimestampScalar(col 1:timestamp, val 1974-10-04 17:21:03.989) -> 38:boolean, VectorUDFYearTimestamp(col 1:timestamp, field YEAR) -> 39:int, VectorUDFYearTimestamp(col 3:timestamp, field YEAR) -> 40:int) -> 41:int, IfExprStringGroupColumnStringScalar(col 42:boolean, col 2:string, val 2018-03-08 23:04:59.000000000)(children: SelectStringColLikeStringScalar(col 2:string) -> 42:boolean) -> 43:string, IfExprNullCondExpr(col 44:boolean, null, col 45:int)(children: TimestampColEqualTimestampScalar(col 1:timestamp, val 2021-09-24 03:18:32.413655165) -> 44:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 45:int) -> 46:int, IfExprCondExprNull(col 49:boolean, col 50:int, null)(children: ColAndCol(col 47:boolean, col 48:boolean)(children: TimestampColGreaterEqualTimestampScalar(col 3:timestamp, val 5344-10-04 18:40:08.165) -> 47:boolean, TimestampColLessTimestampScalar(col 3:timestamp, val 6631-11-13 16:31:29.702202248) -> 48:boolean) -> 49:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 50:int) -> 51:int, IfExprCondExprCondExpr(col 54:boolean, col 55:date, col 56:date)(children: DoubleColGreaterDoubleScalar(col 53:double, val 100.0)(children: DoubleColModuloDoubleScalar(col 52:double, val 500.0)(children: CastTimestampToDouble(col 1:timestamp) -> 52:double) -> 53:double) -> 54:boolean, VectorUDFDateAddColScalar(col 0:date, val 1) -> 55:date, VectorUDFDateAddColScalar(col 0:date, val 365) -> 56:date) -> 57:date + Statistics: Num rows: 51 Data size: 23862 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: timestamp), _col10 (type: string), _col1 (type: timestamp) null sort order: zzz @@ -742,7 +742,7 @@ STAGE PLANS: native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true valueColumns: 16:string, 28:string, 37:string, 41:int, 43:string, 46:int, 51:int, 57:date - Statistics: Num rows: 51 Data size: 23760 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 51 Data size: 23862 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int), _col8 (type: int), _col9 (type: date) Execution mode: vectorized, llap LLAP IO: all inputs @@ -784,13 +784,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 2, 3, 4, 5, 6, 7, 8, 9, 10] - Statistics: Num rows: 51 Data size: 18099 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 51 Data size: 18201 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 51 Data size: 18099 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 51 Data size: 18201 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -870,11 +870,11 @@ POSTHOOK: Input: default@timestamps #### A masked pattern was here #### ctimestamp1 ctimestamp2 ctimestamp2_description ctimestamp2_description_2 ctimestamp2_description_3 field1 field_2 field_3 field_4 field_5 0004-09-22 18:26:29.519542222 0004-09-21 16:23:25.519542222 1800s or Earlier Old Old 4 0004-09-22 18:26:29.519542222 26 NULL 0005-09-22 -0528-10-27 08:15:18.941718273 0528-10-26 06:12:14.941718273 1800s or Earlier Old Old 528 2018-03-08 23:04:59 15 NULL 0529-10-27 +0528-10-27 08:15:18.941718273 0528-10-26 06:12:14.941718273 1800s or Earlier Old Old 528 2018-03-08 23:04:59.000000000 15 NULL 0529-10-27 1319-02-02 16:31:57.778 1319-02-01 14:28:53.778 1800s or Earlier Old Old 1319 1319-02-02 16:31:57.778 31 NULL 1320-02-02 -1404-07-23 15:32:16.059185026 1404-07-22 13:29:12.059185026 1800s or Earlier Old Old 1404 2018-03-08 23:04:59 32 NULL 1405-07-23 -1815-05-06 00:12:37.543584705 1815-05-04 22:09:33.543584705 1900s Old Old 1815 2018-03-08 23:04:59 12 NULL 1816-05-05 -1883-04-17 04:14:34.647766229 1883-04-16 02:11:30.647766229 1900s Old Old 1883 2018-03-08 23:04:59 14 NULL 1884-04-16 +1404-07-23 15:32:16.059185026 1404-07-22 13:29:12.059185026 1800s or Earlier Old Old 1404 2018-03-08 23:04:59.000000000 32 NULL 1405-07-23 +1815-05-06 00:12:37.543584705 1815-05-04 22:09:33.543584705 1900s Old Old 1815 2018-03-08 23:04:59.000000000 12 NULL 1816-05-05 +1883-04-17 04:14:34.647766229 1883-04-16 02:11:30.647766229 1900s Old Old 1883 2018-03-08 23:04:59.000000000 14 NULL 1884-04-16 1966-08-16 13:36:50.183618031 1966-08-15 11:33:46.183618031 Early 2010s Old Old 1966 1966-08-16 13:36:50.183618031 36 NULL 1967-08-16 1973-04-17 06:30:38.596784156 1973-04-16 04:27:34.596784156 Early 2010s Old Old 1973 1973-04-17 06:30:38.596784156 30 NULL 1973-04-18 1974-10-04 17:21:03.989 1974-10-03 15:17:59.989 Early 2010s Old Old 1974 1974-10-04 17:21:03.989 21 NULL 1974-10-05 @@ -889,37 +889,37 @@ ctimestamp1 ctimestamp2 ctimestamp2_description ctimestamp2_description_2 ctimes 1987-05-28 13:52:07.900916635 1987-05-27 11:49:03.900916635 Early 2010s Old Old 1987 1987-05-28 13:52:07.900916635 52 NULL 1987-05-29 1998-10-16 20:05:29.397591987 1998-10-15 18:02:25.397591987 Early 2010s Old Old 1998 1998-10-16 20:05:29.397591987 5 NULL 1998-10-17 1999-10-03 16:59:10.396903939 1999-10-02 14:56:06.396903939 Early 2010s Old Old 1999 1999-10-03 16:59:10.396903939 59 NULL 1999-10-04 -2000-12-18 08:42:30.000595596 2000-12-17 06:39:26.000595596 Early 2010s Old Old 2000 2018-03-08 23:04:59 42 NULL 2000-12-19 -2002-05-10 05:29:48.990818073 2002-05-09 03:26:44.990818073 Early 2010s Early 2000s Early 2000s 2002 2018-03-08 23:04:59 29 NULL 2003-05-10 -2003-09-23 22:33:17.00003252 2003-09-22 20:30:13.00003252 Early 2010s Early 2000s Early 2000s 2003 2018-03-08 23:04:59 33 NULL 2003-09-24 -2004-03-07 20:14:13 2004-03-06 18:11:09 Early 2010s Early 2000s Early 2000s 2004 2018-03-08 23:04:59 14 NULL 2004-03-08 -2007-02-09 05:17:29.368756876 2007-02-08 03:14:25.368756876 Late 2000s Late 2000s Late 2000s 2007 2018-03-08 23:04:59 17 NULL 2007-02-10 -2009-01-21 10:49:07.108 2009-01-20 08:46:03.108 Late 2000s Late 2000s Late 2000s 2009 2018-03-08 23:04:59 49 NULL 2009-01-22 -2010-04-08 02:43:35.861742727 2010-04-07 00:40:31.861742727 Late 2000s Late 2000s Late 2000s 2010 2018-03-08 23:04:59 43 NULL 2010-04-09 -2013-04-07 02:44:43.00086821 2013-04-06 00:41:39.00086821 Early 2010s Early 2010s NULL 2013 2018-03-08 23:04:59 44 NULL 2013-04-08 -2013-04-10 00:43:46.854731546 2013-04-08 22:40:42.854731546 Early 2010s Early 2010s NULL 2013 2018-03-08 23:04:59 43 NULL 2013-04-11 -2021-09-24 03:18:32.413655165 2021-09-23 01:15:28.413655165 Unknown NULL NULL 2021 2018-03-08 23:04:59 NULL NULL 2022-09-24 -2024-11-11 16:42:41.101 2024-11-10 14:39:37.101 Unknown NULL NULL 2024 2018-03-08 23:04:59 42 NULL 2024-11-12 -4143-07-08 10:53:27.252802259 4143-07-07 08:50:23.252802259 Unknown NULL NULL 4143 2018-03-08 23:04:59 53 NULL 4143-07-09 -4966-12-04 09:30:55.202 4966-12-03 07:27:51.202 Unknown NULL NULL 4966 2018-03-08 23:04:59 30 NULL 4966-12-05 -5339-02-01 14:10:01.085678691 5339-01-31 12:06:57.085678691 Unknown NULL NULL 5339 2018-03-08 23:04:59 10 NULL 5339-02-02 -5344-10-04 18:40:08.165 5344-10-03 16:37:04.165 Unknown NULL NULL 5344 2018-03-08 23:04:59 40 NULL 5344-10-05 -5397-07-13 07:12:32.000896438 5397-07-12 05:09:28.000896438 Unknown NULL NULL 5397 2018-03-08 23:04:59 12 12 5397-07-14 -5966-07-09 03:30:50.597 5966-07-08 01:27:46.597 Unknown NULL NULL 5966 2018-03-08 23:04:59 30 30 5966-07-10 -6229-06-28 02:54:28.970117179 6229-06-27 00:51:24.970117179 Unknown NULL NULL 6229 2018-03-08 23:04:59 54 54 6229-06-29 -6482-04-27 12:07:38.073915413 6482-04-26 10:04:34.073915413 Unknown NULL NULL 6482 2018-03-08 23:04:59 7 7 6483-04-27 -6631-11-13 16:31:29.702202248 6631-11-12 14:28:25.702202248 Unknown NULL NULL 6631 2018-03-08 23:04:59 31 31 6631-11-14 -6705-09-28 18:27:28.000845672 6705-09-27 16:24:24.000845672 Unknown NULL NULL 6705 2018-03-08 23:04:59 27 NULL 6705-09-29 -6731-02-12 08:12:48.287783702 6731-02-11 06:09:44.287783702 Unknown NULL NULL 6731 2018-03-08 23:04:59 12 NULL 6731-02-13 -7160-12-02 06:00:24.81200852 7160-12-01 03:57:20.81200852 Unknown NULL NULL 7160 2018-03-08 23:04:59 0 NULL 7160-12-03 -7409-09-07 23:33:32.459349602 7409-09-06 21:30:28.459349602 Unknown NULL NULL 7409 2018-03-08 23:04:59 33 NULL 7410-09-07 -7503-06-23 23:14:17.486 7503-06-22 21:11:13.486 Unknown NULL NULL 7503 2018-03-08 23:04:59 14 NULL 7503-06-24 -8422-07-22 03:21:45.745036084 8422-07-21 01:18:41.745036084 Unknown NULL NULL 8422 2018-03-08 23:04:59 21 NULL 8422-07-23 -8521-01-16 20:42:05.668832388 8521-01-15 18:39:01.668832388 Unknown NULL NULL 8521 2018-03-08 23:04:59 42 NULL 8521-01-17 -9075-06-13 16:20:09.218517797 9075-06-12 14:17:05.218517797 Unknown NULL NULL 9075 2018-03-08 23:04:59 20 NULL 9075-06-14 -9209-11-11 04:08:58.223768453 9209-11-10 02:05:54.223768453 Unknown NULL NULL 9209 2018-03-08 23:04:59 8 NULL 9209-11-12 -9403-01-09 18:12:33.547 9403-01-08 16:09:29.547 Unknown NULL NULL 9403 2018-03-08 23:04:59 12 NULL 9404-01-09 -NULL NULL Unknown NULL NULL NULL 2018-03-08 23:04:59 NULL NULL NULL +2000-12-18 08:42:30.000595596 2000-12-17 06:39:26.000595596 Early 2010s Old Old 2000 2018-03-08 23:04:59.000000000 42 NULL 2000-12-19 +2002-05-10 05:29:48.990818073 2002-05-09 03:26:44.990818073 Early 2010s Early 2000s Early 2000s 2002 2018-03-08 23:04:59.000000000 29 NULL 2003-05-10 +2003-09-23 22:33:17.00003252 2003-09-22 20:30:13.00003252 Early 2010s Early 2000s Early 2000s 2003 2018-03-08 23:04:59.000000000 33 NULL 2003-09-24 +2004-03-07 20:14:13 2004-03-06 18:11:09 Early 2010s Early 2000s Early 2000s 2004 2018-03-08 23:04:59.000000000 14 NULL 2004-03-08 +2007-02-09 05:17:29.368756876 2007-02-08 03:14:25.368756876 Late 2000s Late 2000s Late 2000s 2007 2018-03-08 23:04:59.000000000 17 NULL 2007-02-10 +2009-01-21 10:49:07.108 2009-01-20 08:46:03.108 Late 2000s Late 2000s Late 2000s 2009 2018-03-08 23:04:59.000000000 49 NULL 2009-01-22 +2010-04-08 02:43:35.861742727 2010-04-07 00:40:31.861742727 Late 2000s Late 2000s Late 2000s 2010 2018-03-08 23:04:59.000000000 43 NULL 2010-04-09 +2013-04-07 02:44:43.00086821 2013-04-06 00:41:39.00086821 Early 2010s Early 2010s NULL 2013 2018-03-08 23:04:59.000000000 44 NULL 2013-04-08 +2013-04-10 00:43:46.854731546 2013-04-08 22:40:42.854731546 Early 2010s Early 2010s NULL 2013 2018-03-08 23:04:59.000000000 43 NULL 2013-04-11 +2021-09-24 03:18:32.413655165 2021-09-23 01:15:28.413655165 Unknown NULL NULL 2021 2018-03-08 23:04:59.000000000 NULL NULL 2022-09-24 +2024-11-11 16:42:41.101 2024-11-10 14:39:37.101 Unknown NULL NULL 2024 2018-03-08 23:04:59.000000000 42 NULL 2024-11-12 +4143-07-08 10:53:27.252802259 4143-07-07 08:50:23.252802259 Unknown NULL NULL 4143 2018-03-08 23:04:59.000000000 53 NULL 4143-07-09 +4966-12-04 09:30:55.202 4966-12-03 07:27:51.202 Unknown NULL NULL 4966 2018-03-08 23:04:59.000000000 30 NULL 4966-12-05 +5339-02-01 14:10:01.085678691 5339-01-31 12:06:57.085678691 Unknown NULL NULL 5339 2018-03-08 23:04:59.000000000 10 NULL 5339-02-02 +5344-10-04 18:40:08.165 5344-10-03 16:37:04.165 Unknown NULL NULL 5344 2018-03-08 23:04:59.000000000 40 NULL 5344-10-05 +5397-07-13 07:12:32.000896438 5397-07-12 05:09:28.000896438 Unknown NULL NULL 5397 2018-03-08 23:04:59.000000000 12 12 5397-07-14 +5966-07-09 03:30:50.597 5966-07-08 01:27:46.597 Unknown NULL NULL 5966 2018-03-08 23:04:59.000000000 30 30 5966-07-10 +6229-06-28 02:54:28.970117179 6229-06-27 00:51:24.970117179 Unknown NULL NULL 6229 2018-03-08 23:04:59.000000000 54 54 6229-06-29 +6482-04-27 12:07:38.073915413 6482-04-26 10:04:34.073915413 Unknown NULL NULL 6482 2018-03-08 23:04:59.000000000 7 7 6483-04-27 +6631-11-13 16:31:29.702202248 6631-11-12 14:28:25.702202248 Unknown NULL NULL 6631 2018-03-08 23:04:59.000000000 31 31 6631-11-14 +6705-09-28 18:27:28.000845672 6705-09-27 16:24:24.000845672 Unknown NULL NULL 6705 2018-03-08 23:04:59.000000000 27 NULL 6705-09-29 +6731-02-12 08:12:48.287783702 6731-02-11 06:09:44.287783702 Unknown NULL NULL 6731 2018-03-08 23:04:59.000000000 12 NULL 6731-02-13 +7160-12-02 06:00:24.81200852 7160-12-01 03:57:20.81200852 Unknown NULL NULL 7160 2018-03-08 23:04:59.000000000 0 NULL 7160-12-03 +7409-09-07 23:33:32.459349602 7409-09-06 21:30:28.459349602 Unknown NULL NULL 7409 2018-03-08 23:04:59.000000000 33 NULL 7410-09-07 +7503-06-23 23:14:17.486 7503-06-22 21:11:13.486 Unknown NULL NULL 7503 2018-03-08 23:04:59.000000000 14 NULL 7503-06-24 +8422-07-22 03:21:45.745036084 8422-07-21 01:18:41.745036084 Unknown NULL NULL 8422 2018-03-08 23:04:59.000000000 21 NULL 8422-07-23 +8521-01-16 20:42:05.668832388 8521-01-15 18:39:01.668832388 Unknown NULL NULL 8521 2018-03-08 23:04:59.000000000 42 NULL 8521-01-17 +9075-06-13 16:20:09.218517797 9075-06-12 14:17:05.218517797 Unknown NULL NULL 9075 2018-03-08 23:04:59.000000000 20 NULL 9075-06-14 +9209-11-11 04:08:58.223768453 9209-11-10 02:05:54.223768453 Unknown NULL NULL 9209 2018-03-08 23:04:59.000000000 8 NULL 9209-11-12 +9403-01-09 18:12:33.547 9403-01-08 16:09:29.547 Unknown NULL NULL 9403 2018-03-08 23:04:59.000000000 12 NULL 9404-01-09 +NULL NULL Unknown NULL NULL NULL 2018-03-08 23:04:59.000000000 NULL NULL NULL PREHOOK: query: create temporary table foo(q548284 int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default From cf921ae9422fd7ba9f5335adcc26d62c27bbbd96 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Wed, 9 Sep 2026 11:04:39 +0100 Subject: [PATCH 19/24] Adjust test plans (simplifications probably from CALCITE-7722) --- .../positive/dynamic_partition_pruning.q.out | 91 +++++++-------- .../llap/dynamic_partition_pruning.q.out | 107 ++++++++---------- .../clientpositive/llap/lineage2.q.out | 2 +- .../clientpositive/llap/lineage3.q.out | 4 +- .../clientpositive/llap/vector_coalesce.q.out | 6 +- 5 files changed, 96 insertions(+), 114 deletions(-) diff --git a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out index 94c1e9cb2b82..965fc4967cf6 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out @@ -1467,31 +1467,28 @@ STAGE PLANS: expressions: CAST( UDFToInteger((hr / 2.0D)) AS STRING) (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + minReductionHashAggr: 0.4 + mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - minReductionHashAggr: 0.4 - mode: hash - outputColumnNames: _col0 + Dynamic Partitioning Event Operator + Target column: hr (string) + Target Input: srcpart_iceberg + Partition key expr: hr Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Dynamic Partitioning Event Operator - Target column: hr (string) - Target Input: srcpart_iceberg - Partition key expr: hr - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 + Target Vertex: Map 1 Execution mode: vectorized Reducer 2 Reduce Operator Tree: @@ -1599,15 +1596,12 @@ STAGE PLANS: expressions: CAST( UDFToInteger((hr / 2.0D)) AS STRING) (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Reducer 2 Reduce Operator Tree: @@ -3495,31 +3489,28 @@ STAGE PLANS: expressions: CAST( UDFToInteger((hr / 2.0D)) AS STRING) (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + minReductionHashAggr: 0.4 + mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - minReductionHashAggr: 0.4 - mode: hash - outputColumnNames: _col0 + Dynamic Partitioning Event Operator + Target column: hr (string) + Target Input: srcpart_iceberg + Partition key expr: hr Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Dynamic Partitioning Event Operator - Target column: hr (string) - Target Input: srcpart_iceberg - Partition key expr: hr - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 + Target Vertex: Map 1 Execution mode: vectorized Reducer 2 Execution mode: vectorized diff --git a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out index 924f7c6e95b2..35a35edaa869 100644 --- a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out @@ -1894,31 +1894,28 @@ STAGE PLANS: expressions: UDFToDouble(UDFToInteger((hr / 2.0D))) (type: double) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: double) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: double) + Select Operator + expressions: _col0 (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: double) + minReductionHashAggr: 0.4 + mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: double) - minReductionHashAggr: 0.4 - mode: hash - outputColumnNames: _col0 + Dynamic Partitioning Event Operator + Target column: hr (string) + Target Input: srcpart + Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Dynamic Partitioning Event Operator - Target column: hr (string) - Target Input: srcpart - Partition key expr: UDFToDouble(hr) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 + Target Vertex: Map 1 Execution mode: llap LLAP IO: all inputs Reducer 2 @@ -2018,10 +2015,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (UDFToDouble(hr) * 2.0D) is not null (type: boolean) + filterExpr: UDFToDouble(hr) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (UDFToDouble(hr) * 2.0D) is not null (type: boolean) + predicate: UDFToDouble(hr) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: (UDFToDouble(hr) * 2.0D) (type: double) @@ -2199,15 +2196,12 @@ STAGE PLANS: expressions: UDFToDouble(UDFToInteger((hr / 2.0D))) (type: double) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: double) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs Reducer 2 @@ -2307,10 +2301,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (UDFToDouble(hr) * 2.0D) is not null (type: boolean) + filterExpr: UDFToDouble(hr) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (UDFToDouble(hr) * 2.0D) is not null (type: boolean) + predicate: UDFToDouble(hr) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: (UDFToDouble(hr) * 2.0D) (type: double) @@ -2455,10 +2449,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (UDFToDouble(hr) * 2.0D) is not null (type: boolean) + filterExpr: UDFToDouble(hr) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (UDFToDouble(hr) * 2.0D) is not null (type: boolean) + predicate: UDFToDouble(hr) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: CAST( (UDFToDouble(hr) * 2.0D) AS STRING) (type: string) @@ -5360,31 +5354,28 @@ STAGE PLANS: expressions: UDFToDouble(UDFToInteger((hr / 2.0D))) (type: double) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: double) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: double) + Select Operator + expressions: _col0 (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: double) + minReductionHashAggr: 0.4 + mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: double) - minReductionHashAggr: 0.4 - mode: hash - outputColumnNames: _col0 + Dynamic Partitioning Event Operator + Target column: hr (string) + Target Input: srcpart + Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Dynamic Partitioning Event Operator - Target column: hr (string) - Target Input: srcpart - Partition key expr: UDFToDouble(hr) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 + Target Vertex: Map 1 Execution mode: llap LLAP IO: all inputs Reducer 2 @@ -5463,10 +5454,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (UDFToDouble(hr) * 2.0D) is not null (type: boolean) + filterExpr: UDFToDouble(hr) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (UDFToDouble(hr) * 2.0D) is not null (type: boolean) + predicate: UDFToDouble(hr) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: (UDFToDouble(hr) * 2.0D) (type: double) diff --git a/ql/src/test/results/clientpositive/llap/lineage2.q.out b/ql/src/test/results/clientpositive/llap/lineage2.q.out index a2980d5561a6..7af56737e8e0 100644 --- a/ql/src/test/results/clientpositive/llap/lineage2.q.out +++ b/ql/src/test/results/clientpositive/llap/lineage2.q.out @@ -490,7 +490,7 @@ PREHOOK: Input: default@src1 PREHOOK: Input: default@src2 PREHOOK: Output: default@dest2_n11 Result schema has 4 fields, but we don't get as many dependencies -{"version":"1.0","engine":"tez","database":"default","hash":"4c13fe982c4d22e5735ba469dee4b3d8","queryText":"insert into table dest2_n11\n select * from src1 JOIN src2 ON length(src1.value) = length(src2.value2) + 1","edges":[{"sources":[4],"targets":[0],"edgeType":"PROJECTION"},{"sources":[5],"targets":[1],"edgeType":"PROJECTION"},{"sources":[6],"targets":[2],"edgeType":"PROJECTION"},{"sources":[7],"targets":[3],"edgeType":"PROJECTION"},{"sources":[5],"targets":[0,1,2,3],"expression":"length(src1.value) is not null","edgeType":"PREDICATE"},{"sources":[5,7],"targets":[0,1,2,3],"expression":"(length(src1.value) = (length(src2.value2) + 1))","edgeType":"PREDICATE"},{"sources":[7],"targets":[0,1,2,3],"expression":"(length(src2.value2) + 1) is not null","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest2_n11.key"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest2_n11.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest2_n11.key2"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest2_n11.value2"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":5,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":6,"vertexType":"COLUMN","vertexId":"default.src2.key2"},{"id":7,"vertexType":"COLUMN","vertexId":"default.src2.value2"}]} +{"version":"1.0","engine":"tez","database":"default","hash":"4c13fe982c4d22e5735ba469dee4b3d8","queryText":"insert into table dest2_n11\n select * from src1 JOIN src2 ON length(src1.value) = length(src2.value2) + 1","edges":[{"sources":[4],"targets":[0],"edgeType":"PROJECTION"},{"sources":[5],"targets":[1],"edgeType":"PROJECTION"},{"sources":[6],"targets":[2],"edgeType":"PROJECTION"},{"sources":[7],"targets":[3],"edgeType":"PROJECTION"},{"sources":[5],"targets":[0,1,2,3],"expression":"length(src1.value) is not null","edgeType":"PREDICATE"},{"sources":[5,7],"targets":[0,1,2,3],"expression":"(length(src1.value) = (length(src2.value2) + 1))","edgeType":"PREDICATE"},{"sources":[7],"targets":[0,1,2,3],"expression":"length(src2.value2) is not null","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest2_n11.key"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest2_n11.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest2_n11.key2"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest2_n11.value2"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":5,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":6,"vertexType":"COLUMN","vertexId":"default.src2.key2"},{"id":7,"vertexType":"COLUMN","vertexId":"default.src2.value2"}]} PREHOOK: query: select * from src1 where length(key) > 2 PREHOOK: type: QUERY PREHOOK: Input: default@src1 diff --git a/ql/src/test/results/clientpositive/llap/lineage3.q.out b/ql/src/test/results/clientpositive/llap/lineage3.q.out index 5c0920c6717a..dbc622a30305 100644 --- a/ql/src/test/results/clientpositive/llap/lineage3.q.out +++ b/ql/src/test/results/clientpositive/llap/lineage3.q.out @@ -176,7 +176,7 @@ where key in (select key+18 from src1) order by key PREHOOK: type: QUERY PREHOOK: Input: default@src1 #### A masked pattern was here #### -{"version":"1.0","engine":"tez","database":"default","hash":"06c63ecdebcc1ca975a34b0fe1b4bf38","queryText":"select key, value from src1\nwhere key in (select key+18 from src1) order by key","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"UDFToDouble(src1.key) is not null","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) = (UDFToDouble(src1.key) + 18.0D))","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) + 18.0D) is not null","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"key"},{"id":1,"vertexType":"COLUMN","vertexId":"value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"}]} +{"version":"1.0","engine":"tez","database":"default","hash":"06c63ecdebcc1ca975a34b0fe1b4bf38","queryText":"select key, value from src1\nwhere key in (select key+18 from src1) order by key","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"UDFToDouble(src1.key) is not null","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) = (UDFToDouble(src1.key) + 18.0D))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"key"},{"id":1,"vertexType":"COLUMN","vertexId":"value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"}]} 146 val_146 273 val_273 PREHOOK: query: select * from src1 a @@ -196,7 +196,7 @@ where key not in (select key+18 from src1) order by key PREHOOK: type: QUERY PREHOOK: Input: default@src1 #### A masked pattern was here #### -{"version":"1.0","engine":"tez","database":"default","hash":"cbc4367150807328dda0f1cf4c74b811","queryText":"select key, value from src1\nwhere key not in (select key+18 from src1) order by key","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) = (UDFToDouble(src1.key) + 18.0D))","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) + 18.0D) is not null","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"((count(*) = 0L) or (true is null and (count((UDFToDouble(src1.key) + 18.0D)) >= count(*)) and src1.key is not null))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"key"},{"id":1,"vertexType":"COLUMN","vertexId":"value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"TABLE","vertexId":"default.src1"}]} +{"version":"1.0","engine":"tez","database":"default","hash":"cbc4367150807328dda0f1cf4c74b811","queryText":"select key, value from src1\nwhere key not in (select key+18 from src1) order by key","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) = (UDFToDouble(src1.key) + 18.0D))","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"UDFToDouble(src1.key) is not null","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"((count(*) = 0L) or (true is null and (count((UDFToDouble(src1.key) + 18.0D)) >= count(*)) and src1.key is not null))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"key"},{"id":1,"vertexType":"COLUMN","vertexId":"value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"TABLE","vertexId":"default.src1"}]} PREHOOK: query: select * from src1 a where not exists (select cint from alltypesorc b diff --git a/ql/src/test/results/clientpositive/llap/vector_coalesce.q.out b/ql/src/test/results/clientpositive/llap/vector_coalesce.q.out index b44bc3381805..188b71359225 100644 --- a/ql/src/test/results/clientpositive/llap/vector_coalesce.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_coalesce.q.out @@ -154,13 +154,13 @@ STAGE PLANS: predicateExpression: SelectColumnIsNull(col 0:tinyint) Top N Key Vectorization: className: VectorTopNKeyOperator - keyExpressions: col 5:double, col 2:int, IfExprCondExprColumn(col 16:boolean, col 18:double, col 19:double)(children: IsNotNull(col 15:double)(children: DoubleColAddDoubleColumn(col 5:double, col 14:double)(children: FuncLog2LongToDouble(col 2:int) -> 14:double) -> 15:double) -> 16:boolean, DoubleColAddDoubleColumn(col 5:double, col 17:double)(children: FuncLog2LongToDouble(col 2:int) -> 17:double) -> 18:double, ConstantVectorExpression(val 0.0) -> 19:double) -> 20:double + keyExpressions: col 5:double, col 2:int, IfExprCondExprColumn(col 17:boolean, col 19:double, col 20:double)(children: ColAndCol(col 14:boolean, col 16:boolean)(children: IsNotNull(col 5:double) -> 14:boolean, IsNotNull(col 15:double)(children: FuncLog2LongToDouble(col 2:int) -> 15:double) -> 16:boolean) -> 17:boolean, DoubleColAddDoubleColumn(col 5:double, col 18:double)(children: FuncLog2LongToDouble(col 2:int) -> 18:double) -> 19:double, ConstantVectorExpression(val 0.0) -> 20:double) -> 21:double native: true Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [5, 2, 17] - selectExpressions: IfExprCondExprColumn(col 21:boolean, col 15:double, col 14:double)(children: IsNotNull(col 15:double)(children: DoubleColAddDoubleColumn(col 5:double, col 14:double)(children: FuncLog2LongToDouble(col 2:int) -> 14:double) -> 15:double) -> 21:boolean, DoubleColAddDoubleColumn(col 5:double, col 14:double)(children: FuncLog2LongToDouble(col 2:int) -> 14:double) -> 15:double, ConstantVectorExpression(val 0.0) -> 14:double) -> 17:double + projectedOutputColumnNums: [5, 2, 23] + selectExpressions: IfExprCondExprColumn(col 22:boolean, col 18:double, col 15:double)(children: ColAndCol(col 14:boolean, col 16:boolean)(children: IsNotNull(col 5:double) -> 14:boolean, IsNotNull(col 15:double)(children: FuncLog2LongToDouble(col 2:int) -> 15:double) -> 16:boolean) -> 22:boolean, DoubleColAddDoubleColumn(col 5:double, col 15:double)(children: FuncLog2LongToDouble(col 2:int) -> 15:double) -> 18:double, ConstantVectorExpression(val 0.0) -> 15:double) -> 23:double Reduce Sink Vectorization: className: VectorReduceSinkObjectHashOperator native: true From 59bf193fa49bcf22c630ded263876b3ceaff04e4 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Wed, 9 Sep 2026 14:23:55 +0100 Subject: [PATCH 20/24] Adjust test plans (simplifications probably from CALCITE-7722) --- ql/src/test/results/clientpositive/llap/explainuser_1.q.out | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out index 4e6c1d4bdc90..62428aa63464 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -6562,7 +6562,7 @@ Stage-0 <-Select Operator [SEL_5] (rows=500 width=12) Output:["_col0","_col1"] Filter Operator [FIL_19] (rows=500 width=87) - predicate:(UDFToDouble(key) + 1.0D) is not null + predicate:UDFToDouble(key) is not null TableScan [TS_3] (rows=500 width=87) default@src,c,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] @@ -6813,7 +6813,7 @@ Stage-0 Select Operator [SEL_4] (rows=1 width=184) Output:["_col0","_col1"] Filter Operator [FIL_17] (rows=1 width=184) - predicate:(UDFToDouble(key) + 1.0D) is not null + predicate:UDFToDouble(key) is not null TableScan [TS_2] (rows=1 width=184) default@t2_n70,c,Tbl:COMPLETE,Col:NONE,Output:["key"] <-Select Operator [SEL_1] (rows=1 width=368) From 77aa07e848e13132d7c1eb7a6b5591198b71ea39 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Wed, 9 Sep 2026 18:02:57 +0100 Subject: [PATCH 21/24] AlterMaterializedViewRebuildAnalyzer fix + adjust test files mv_iceberg_orc2/4/5/6/7.q.out --- .../results/positive/mv_iceberg_orc2.q.out | 2 +- .../results/positive/mv_iceberg_orc4.q.out | 359 ++++-------------- .../results/positive/mv_iceberg_orc5.q.out | 276 ++++---------- .../results/positive/mv_iceberg_orc6.q.out | 8 +- .../results/positive/mv_iceberg_orc7.q.out | 144 +------ .../AlterMaterializedViewRebuildAnalyzer.java | 29 ++ 6 files changed, 201 insertions(+), 617 deletions(-) diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc2.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc2.q.out index 66dcca6edcf2..ddc4dcf4d679 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc2.q.out @@ -273,7 +273,7 @@ POSTHOOK: Output: default@mat1 CBO PLAN: HiveProject(tbl_ice.b=[$1], tbl_ice.c=[$2]) HiveFilter(condition=[>($2, 52)]) - HiveTableScan(table=[[default, tbl_ice]], table:alias=[tbl_ice], fromVersion=[#Masked#]) + HiveTableScan(table=[[default, tbl_ice]], table:alias=[tbl_ice]) PREHOOK: query: alter materialized view mat1 rebuild PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc4.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc4.q.out index 5fbd6803a7c9..3d2fff8b0542 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc4.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc4.q.out @@ -80,45 +80,35 @@ POSTHOOK: Output: default@tbl_ice_v2 PREHOOK: query: explain cbo alter materialized view mat1 rebuild PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -PREHOOK: Input: default@mat1 PREHOOK: Input: default@tbl_ice PREHOOK: Input: default@tbl_ice_v2 PREHOOK: Output: default@mat1 POSTHOOK: query: explain cbo alter materialized view mat1 rebuild POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -POSTHOOK: Input: default@mat1 POSTHOOK: Input: default@tbl_ice POSTHOOK: Input: default@tbl_ice_v2 POSTHOOK: Output: default@mat1 CBO PLAN: HiveProject(tbl_ice.b=[$0], tbl_ice.c=[$1], _c2=[$2]) - HiveAggregate(group=[{0, 1}], agg#0=[sum($2)]) - HiveProject(b=[$0], c=[$1], $f2=[$2]) - HiveUnion(all=[true]) - HiveProject(b=[$0], c=[$1], $f2=[$2]) - HiveAggregate(group=[{1, 2}], agg#0=[sum($4)]) - HiveJoin(condition=[=($0, $3)], joinType=[inner], algorithm=[none], cost=[not available]) - HiveProject(a=[$0], b=[$1], c=[$2]) - HiveFilter(condition=[AND(>($2, 52), IS NOT NULL($0))]) - HiveTableScan(table=[[default, tbl_ice]], table:alias=[tbl_ice], fromVersion=[#Masked#]) - HiveProject(d=[$0], f=[$2]) - HiveFilter(condition=[IS NOT NULL($0)]) - HiveTableScan(table=[[default, tbl_ice_v2]], table:alias=[tbl_ice_v2], fromVersion=[#Masked#]) - HiveProject(b=[$0], c=[$1], _c2=[$2]) - HiveTableScan(table=[[default, mat1]], table:alias=[default.mat1]) + HiveAggregate(group=[{1, 2}], agg#0=[sum($4)]) + HiveJoin(condition=[=($0, $3)], joinType=[inner], algorithm=[none], cost=[not available]) + HiveProject(a=[$0], b=[$1], c=[$2]) + HiveFilter(condition=[AND(>($2, 52), IS NOT NULL($0))]) + HiveTableScan(table=[[default, tbl_ice]], table:alias=[tbl_ice]) + HiveProject(d=[$0], f=[$2]) + HiveFilter(condition=[IS NOT NULL($0)]) + HiveTableScan(table=[[default, tbl_ice_v2]], table:alias=[tbl_ice_v2]) PREHOOK: query: explain alter materialized view mat1 rebuild PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -PREHOOK: Input: default@mat1 PREHOOK: Input: default@tbl_ice PREHOOK: Input: default@tbl_ice_v2 PREHOOK: Output: default@mat1 POSTHOOK: query: explain alter materialized view mat1 rebuild POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -POSTHOOK: Input: default@mat1 POSTHOOK: Input: default@tbl_ice POSTHOOK: Input: default@tbl_ice_v2 POSTHOOK: Output: default@mat1 @@ -134,11 +124,9 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 8 <- Union 4 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -147,7 +135,6 @@ STAGE PLANS: alias: tbl_ice filterExpr: ((c > 52) and a is not null) (type: boolean) Statistics: Num rows: 8 Data size: 768 Basic stats: COMPLETE Column stats: COMPLETE - Version interval from: #Masked# Filter Operator predicate: ((c > 52) and a is not null) (type: boolean) Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE @@ -163,13 +150,12 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: int) Execution mode: vectorized - Map 7 + Map 5 Map Operator Tree: TableScan alias: tbl_ice_v2 filterExpr: d is not null (type: boolean) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Version interval from: #Masked# Filter Operator predicate: d is not null (type: boolean) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE @@ -185,30 +171,6 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) Execution mode: vectorized - Map 8 - Map Operator Tree: - TableScan - alias: default.mat1 - Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: b (type: string), c (type: int), _c2 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(_col2) - keys: _col0 (type: string), _col1 (type: int) - minReductionHashAggr: 0.4 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: vectorized Reducer 2 Reduce Operator Tree: Merge Join Operator @@ -234,29 +196,6 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) Reducer 3 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(_col2) - keys: _col0 (type: string), _col1 (type: int) - minReductionHashAggr: 0.4 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Reducer 5 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -288,7 +227,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) - Reducer 6 + Reducer 4 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -307,8 +246,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 4 - Vertex: Union 4 Stage: Stage-2 Dependency Collection @@ -339,95 +276,61 @@ STAGE PLANS: PREHOOK: query: explain cbo alter materialized view mat1 rebuild PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -PREHOOK: Input: default@mat1 PREHOOK: Input: default@tbl_ice PREHOOK: Input: default@tbl_ice_v2 PREHOOK: Output: default@mat1 -PREHOOK: Output: default@mat1 POSTHOOK: query: explain cbo alter materialized view mat1 rebuild POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -POSTHOOK: Input: default@mat1 POSTHOOK: Input: default@tbl_ice POSTHOOK: Input: default@tbl_ice_v2 POSTHOOK: Output: default@mat1 -POSTHOOK: Output: default@mat1 CBO PLAN: -HiveProject(tbl_ice.b=[$4], tbl_ice.c=[$5], _c2=[CASE(IS NULL($2), $6, IS NULL($6), $2, +($6, $2))]) - HiveFilter(condition=[OR($3, IS NULL($3))]) - HiveJoin(condition=[AND(IS NOT DISTINCT FROM($0, $4), IS NOT DISTINCT FROM($1, $5))], joinType=[right], algorithm=[BucketJoin], cost=[not available]) - HiveProject(b=[$0], c=[$1], _c2=[$2], $f3=[true]) - HiveTableScan(table=[[default, mat1]], table:alias=[default.mat1]) - HiveProject(b=[$0], c=[$1], $f2=[$2]) - HiveAggregate(group=[{1, 2}], agg#0=[sum($4)]) - HiveJoin(condition=[=($0, $3)], joinType=[inner], algorithm=[CommonJoin], cost=[not available]) - HiveProject(a=[$0], b=[$1], c=[$2]) - HiveFilter(condition=[AND(>($2, 52), IS NOT NULL($0))]) - HiveTableScan(table=[[default, tbl_ice]], table:alias=[tbl_ice], fromVersion=[#Masked#]) - HiveProject(d=[$0], f=[$2]) - HiveFilter(condition=[IS NOT NULL($0)]) - HiveTableScan(table=[[default, tbl_ice_v2]], table:alias=[tbl_ice_v2], fromVersion=[#Masked#]) +HiveProject(tbl_ice.b=[$0], tbl_ice.c=[$1], _c2=[$2]) + HiveAggregate(group=[{1, 2}], agg#0=[sum($4)]) + HiveJoin(condition=[=($0, $3)], joinType=[inner], algorithm=[none], cost=[not available]) + HiveProject(a=[$0], b=[$1], c=[$2]) + HiveFilter(condition=[AND(>($2, 52), IS NOT NULL($0))]) + HiveTableScan(table=[[default, tbl_ice]], table:alias=[tbl_ice]) + HiveProject(d=[$0], f=[$2]) + HiveFilter(condition=[IS NOT NULL($0)]) + HiveTableScan(table=[[default, tbl_ice_v2]], table:alias=[tbl_ice_v2]) PREHOOK: query: explain alter materialized view mat1 rebuild PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -PREHOOK: Input: default@mat1 PREHOOK: Input: default@tbl_ice PREHOOK: Input: default@tbl_ice_v2 PREHOOK: Output: default@mat1 -PREHOOK: Output: default@mat1 POSTHOOK: query: explain alter materialized view mat1 rebuild POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -POSTHOOK: Input: default@mat1 POSTHOOK: Input: default@tbl_ice POSTHOOK: Input: default@tbl_ice_v2 POSTHOOK: Output: default@mat1 -POSTHOOK: Output: default@mat1 STAGE DEPENDENCIES: - Stage-3 is a root stage + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 Stage-4 depends on stages: Stage-3 - Stage-0 depends on stages: Stage-4 - Stage-5 depends on stages: Stage-0 - Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-3 + Stage: Stage-1 Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 - Map Operator Tree: - TableScan - alias: default.mat1 - Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: b (type: string), c (type: int), _c2 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 616 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 2 Data size: 616 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint), _col3 (type: boolean), _col4 (type: int), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint) - Execution mode: vectorized - Map 6 Map Operator Tree: TableScan alias: tbl_ice filterExpr: ((c > 52) and a is not null) (type: boolean) Statistics: Num rows: 8 Data size: 768 Basic stats: COMPLETE Column stats: COMPLETE - Version interval from: #Masked# Filter Operator predicate: ((c > 52) and a is not null) (type: boolean) Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE @@ -443,13 +346,12 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: int) Execution mode: vectorized - Map 9 + Map 5 Map Operator Tree: TableScan alias: tbl_ice_v2 filterExpr: d is not null (type: boolean) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Version interval from: #Masked# Filter Operator predicate: d is not null (type: boolean) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE @@ -469,120 +371,59 @@ STAGE PLANS: Reduce Operator Tree: Merge Join Operator condition map: - Right Outer Join 0 to 1 + Inner Join 0 to 1 keys: - 0 _col0 (type: string), _col1 (type: int) - 1 _col0 (type: string), _col1 (type: int) - nullSafes: [true, true] - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 6 Data size: 1524 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col3 (type: boolean) - Statistics: Num rows: 1 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col4 (type: int), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col0 (type: string), _col1 (type: int), _col2 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: bigint) - Filter Operator - predicate: _col3 (type: boolean) - Statistics: Num rows: 1 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col8 (type: string), _col9 (type: int), CASE WHEN (_col2 is null) THEN (_col10) WHEN (_col10 is null) THEN (_col2) ELSE ((_col10 + _col2)) END (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.mat1 - Select Operator - expressions: _col0 (type: string), _col1 (type: int), _col2 (type: bigint) - outputColumnNames: b, c, _c2 - Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: max(length(b)), avg(COALESCE(length(b),0)), count(1), count(b), compute_bit_vector_hll(b), min(c), max(c), count(c), compute_bit_vector_hll(c), min(_c2), max(_c2), count(_c2), compute_bit_vector_hll(_c2) - minReductionHashAggr: 0.4 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) - Filter Operator - predicate: _col3 is null (type: boolean) - Statistics: Num rows: 4 Data size: 1016 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col8 (type: string), _col9 (type: int), CASE WHEN (_col2 is null) THEN (_col10) WHEN (_col10 is null) THEN (_col2) ELSE ((_col10 + _col2)) END (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.mat1 - Select Operator - expressions: _col0 (type: string), _col1 (type: int), _col2 (type: bigint) - outputColumnNames: b, c, _c2 - Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: max(length(b)), avg(COALESCE(length(b),0)), count(1), count(b), compute_bit_vector_hll(b), min(c), max(c), count(c), compute_bit_vector_hll(c), min(_c2), max(_c2), count(_c2), compute_bit_vector_hll(_c2) - minReductionHashAggr: 0.75 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col1, _col2, _col4 + Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col4) + keys: _col1 (type: string), _col2 (type: int) + minReductionHashAggr: 0.4 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) Reducer 3 Execution mode: vectorized Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.mat1 - Reducer 4 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + expressions: _col0 (type: string), _col1 (type: int), _col2 (type: bigint) + outputColumnNames: b, c, _c2 + Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(length(b)), avg(COALESCE(length(b),0)), count(1), count(b), compute_bit_vector_hll(b), min(c), max(c), count(c), compute_bit_vector_hll(c), min(_c2), max(_c2), count(_c2), compute_bit_vector_hll(_c2) + minReductionHashAggr: 0.75 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) + Reducer 4 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -601,61 +442,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col1, _col2, _col4 - Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(_col4) - keys: _col1 (type: string), _col2 (type: int) - minReductionHashAggr: 0.4 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Reducer 8 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Stage: Stage-4 + Stage: Stage-2 Dependency Collection Stage: Stage-0 Move Operator tables: - replace: false + replace: true table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.mat1 - Stage: Stage-5 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -663,25 +464,21 @@ STAGE PLANS: Column Types: string, int, bigint Table: default.mat1 - Stage: Stage-6 + Stage: Stage-4 Materialized View Update name: default.mat1 update creation metadata: true PREHOOK: query: alter materialized view mat1 rebuild PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -PREHOOK: Input: default@mat1 PREHOOK: Input: default@tbl_ice PREHOOK: Input: default@tbl_ice_v2 PREHOOK: Output: default@mat1 -PREHOOK: Output: default@mat1 POSTHOOK: query: alter materialized view mat1 rebuild POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -POSTHOOK: Input: default@mat1 POSTHOOK: Input: default@tbl_ice POSTHOOK: Input: default@tbl_ice_v2 POSTHOOK: Output: default@mat1 -POSTHOOK: Output: default@mat1 PREHOOK: query: select * from mat1 PREHOOK: type: QUERY PREHOOK: Input: default@mat1 @@ -690,5 +487,5 @@ POSTHOOK: query: select * from mat1 POSTHOOK: type: QUERY POSTHOOK: Input: default@mat1 POSTHOOK: Output: hdfs://### HDFS PATH ### -five 54 108 -four 53 106 +five 54 216 +four 53 212 diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc5.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc5.q.out index dabdc0989f39..689724a769a4 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc5.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc5.q.out @@ -82,95 +82,61 @@ POSTHOOK: Output: default@tbl_ice_v2 PREHOOK: query: explain cbo alter materialized view mat2 rebuild PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -PREHOOK: Input: default@mat2 PREHOOK: Input: default@tbl_ice PREHOOK: Input: default@tbl_ice_v2 PREHOOK: Output: default@mat2 -PREHOOK: Output: default@mat2 POSTHOOK: query: explain cbo alter materialized view mat2 rebuild POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -POSTHOOK: Input: default@mat2 POSTHOOK: Input: default@tbl_ice POSTHOOK: Input: default@tbl_ice_v2 POSTHOOK: Output: default@mat2 -POSTHOOK: Output: default@mat2 CBO PLAN: -HiveProject(tbl_ice.b=[$5], tbl_ice.c=[$6], _c2=[CASE(IS NULL($2), $7, IS NULL($7), $2, +($7, $2))], _c3=[CASE(IS NULL($3), $8, IS NULL($8), $3, +($8, $3))], _c4=[/(CAST(CASE(IS NULL($2), $7, IS NULL($7), $2, +($7, $2))):DOUBLE, CASE(IS NULL($3), $8, IS NULL($8), $3, +($8, $3)))]) - HiveFilter(condition=[OR($4, IS NULL($4))]) - HiveJoin(condition=[AND(IS NOT DISTINCT FROM($0, $5), IS NOT DISTINCT FROM($1, $6))], joinType=[right], algorithm=[BucketJoin], cost=[not available]) - HiveProject(b=[$0], c=[$1], _c2=[$2], _c3=[$3], $f4=[true]) - HiveTableScan(table=[[default, mat2]], table:alias=[default.mat2]) - HiveProject(b=[$0], c=[$1], $f2=[$2], $f3=[$3]) - HiveAggregate(group=[{1, 2}], agg#0=[sum($4)], agg#1=[count($4)]) - HiveJoin(condition=[=($0, $3)], joinType=[inner], algorithm=[CommonJoin], cost=[not available]) - HiveProject(a=[$0], b=[$1], c=[$2]) - HiveFilter(condition=[AND(>($2, 52), IS NOT NULL($0))]) - HiveTableScan(table=[[default, tbl_ice]], table:alias=[tbl_ice], fromVersion=[#Masked#]) - HiveProject(d=[$0], f=[$2]) - HiveFilter(condition=[IS NOT NULL($0)]) - HiveTableScan(table=[[default, tbl_ice_v2]], table:alias=[tbl_ice_v2], fromVersion=[#Masked#]) +HiveProject(tbl_ice.b=[$0], tbl_ice.c=[$1], _c2=[$2], _c3=[$3], _c4=[/(CAST($2):DOUBLE, $3)]) + HiveAggregate(group=[{1, 2}], agg#0=[sum($4)], agg#1=[count($4)]) + HiveJoin(condition=[=($0, $3)], joinType=[inner], algorithm=[none], cost=[not available]) + HiveProject(a=[$0], b=[$1], c=[$2]) + HiveFilter(condition=[AND(>($2, 52), IS NOT NULL($0))]) + HiveTableScan(table=[[default, tbl_ice]], table:alias=[tbl_ice]) + HiveProject(d=[$0], f=[$2]) + HiveFilter(condition=[IS NOT NULL($0)]) + HiveTableScan(table=[[default, tbl_ice_v2]], table:alias=[tbl_ice_v2]) PREHOOK: query: explain alter materialized view mat2 rebuild PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -PREHOOK: Input: default@mat2 PREHOOK: Input: default@tbl_ice PREHOOK: Input: default@tbl_ice_v2 PREHOOK: Output: default@mat2 -PREHOOK: Output: default@mat2 POSTHOOK: query: explain alter materialized view mat2 rebuild POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -POSTHOOK: Input: default@mat2 POSTHOOK: Input: default@tbl_ice POSTHOOK: Input: default@tbl_ice_v2 POSTHOOK: Output: default@mat2 -POSTHOOK: Output: default@mat2 STAGE DEPENDENCIES: - Stage-3 is a root stage + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 Stage-4 depends on stages: Stage-3 - Stage-0 depends on stages: Stage-4 - Stage-5 depends on stages: Stage-0 - Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-3 + Stage: Stage-1 Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 - Map Operator Tree: - TableScan - alias: default.mat2 - Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: b (type: string), c (type: int), _c2 (type: bigint), _c3 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), _c4 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 2 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 2 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint), _col3 (type: bigint), _col4 (type: boolean), _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: double) - Execution mode: vectorized - Map 6 Map Operator Tree: TableScan alias: tbl_ice filterExpr: ((c > 52) and a is not null) (type: boolean) Statistics: Num rows: 8 Data size: 768 Basic stats: COMPLETE Column stats: COMPLETE - Version interval from: #Masked# Filter Operator predicate: ((c > 52) and a is not null) (type: boolean) Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE @@ -186,13 +152,12 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: int) Execution mode: vectorized - Map 9 + Map 5 Map Operator Tree: TableScan alias: tbl_ice_v2 filterExpr: d is not null (type: boolean) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Version interval from: #Masked# Filter Operator predicate: d is not null (type: boolean) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE @@ -212,120 +177,63 @@ STAGE PLANS: Reduce Operator Tree: Merge Join Operator condition map: - Right Outer Join 0 to 1 + Inner Join 0 to 1 keys: - 0 _col0 (type: string), _col1 (type: int) - 1 _col0 (type: string), _col1 (type: int) - nullSafes: [true, true] - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 1620 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col4 (type: boolean) - Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col0 (type: string), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint), _col9 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: double) - Filter Operator - predicate: _col4 (type: boolean) - Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col10 (type: string), _col11 (type: int), CASE WHEN (_col2 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col2) ELSE ((_col12 + _col2)) END (type: bigint), CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END (type: bigint), (UDFToDouble(CASE WHEN (_col2 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col2) ELSE ((_col12 + _col2)) END) / CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.mat2 - Select Operator - expressions: _col0 (type: string), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: double) - outputColumnNames: b, c, _c2, _c3, _c4 - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: max(length(b)), avg(COALESCE(length(b),0)), count(1), count(b), compute_bit_vector_hll(b), min(c), max(c), count(c), compute_bit_vector_hll(c), min(_c2), max(_c2), count(_c2), compute_bit_vector_hll(_c2), min(_c3), max(_c3), count(_c3), compute_bit_vector_hll(_c3), min(_c4), max(_c4), count(_c4), compute_bit_vector_hll(_c4) - minReductionHashAggr: 0.4 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 - Statistics: Num rows: 1 Data size: 904 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 904 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary), _col13 (type: bigint), _col14 (type: bigint), _col15 (type: bigint), _col16 (type: binary), _col17 (type: double), _col18 (type: double), _col19 (type: bigint), _col20 (type: binary) - Filter Operator - predicate: _col4 is null (type: boolean) - Statistics: Num rows: 4 Data size: 1080 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col10 (type: string), _col11 (type: int), CASE WHEN (_col2 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col2) ELSE ((_col12 + _col2)) END (type: bigint), CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END (type: bigint), (UDFToDouble(CASE WHEN (_col2 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col2) ELSE ((_col12 + _col2)) END) / CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col1, _col2, _col4 + Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col4), count(_col4) + keys: _col1 (type: string), _col2 (type: int) + minReductionHashAggr: 0.4 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 4 Data size: 432 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 4 Data size: 432 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 432 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.mat2 - Select Operator - expressions: _col0 (type: string), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: double) - outputColumnNames: b, c, _c2, _c3, _c4 - Statistics: Num rows: 4 Data size: 432 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: max(length(b)), avg(COALESCE(length(b),0)), count(1), count(b), compute_bit_vector_hll(b), min(c), max(c), count(c), compute_bit_vector_hll(c), min(_c2), max(_c2), count(_c2), compute_bit_vector_hll(_c2), min(_c3), max(_c3), count(_c3), compute_bit_vector_hll(_c3), min(_c4), max(_c4), count(_c4), compute_bit_vector_hll(_c4) - minReductionHashAggr: 0.75 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 - Statistics: Num rows: 1 Data size: 904 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 904 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary), _col13 (type: bigint), _col14 (type: bigint), _col15 (type: bigint), _col16 (type: binary), _col17 (type: double), _col18 (type: double), _col19 (type: bigint), _col20 (type: binary) + value expressions: _col2 (type: bigint), _col3 (type: bigint) Reducer 3 - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: bigint), VALUE._col7 (type: bigint), VALUE._col8 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.mat2 - Reducer 4 Execution mode: vectorized Reduce Operator Tree: Group By Operator - aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), min(VALUE._col13), max(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16), min(VALUE._col17), max(VALUE._col18), count(VALUE._col19), compute_bit_vector_hll(VALUE._col20) + aggregations: sum(VALUE._col0), count(VALUE._col1) + keys: KEY._col0 (type: string), KEY._col1 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 - Statistics: Num rows: 1 Data size: 836 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 4 Data size: 432 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'LONG' (type: string), _col13 (type: bigint), _col14 (type: bigint), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary), 'DOUBLE' (type: string), _col17 (type: double), _col18 (type: double), (_col2 - _col19) (type: bigint), COALESCE(ndv_compute_bit_vector(_col20),0) (type: bigint), _col20 (type: binary) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29 - Statistics: Num rows: 1 Data size: 1324 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: string), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint), (UDFToDouble(_col2) / _col3) (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1324 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.mat2 + Select Operator + expressions: _col0 (type: string), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: double) + outputColumnNames: b, c, _c2, _c3, _c4 + Statistics: Num rows: 4 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(length(b)), avg(COALESCE(length(b),0)), count(1), count(b), compute_bit_vector_hll(b), min(c), max(c), count(c), compute_bit_vector_hll(c), min(_c2), max(_c2), count(_c2), compute_bit_vector_hll(_c2), min(_c3), max(_c3), count(_c3), compute_bit_vector_hll(_c3), min(_c4), max(_c4), count(_c4), compute_bit_vector_hll(_c4) + minReductionHashAggr: 0.75 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 + Statistics: Num rows: 1 Data size: 904 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 904 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary), _col13 (type: bigint), _col14 (type: bigint), _col15 (type: bigint), _col16 (type: binary), _col17 (type: double), _col18 (type: double), _col19 (type: bigint), _col20 (type: binary) + Reducer 4 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -344,61 +252,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col1, _col2, _col4 - Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(_col4), count(_col4) - keys: _col1 (type: string), _col2 (type: int) - minReductionHashAggr: 0.4 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 432 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 4 Data size: 432 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 8 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), count(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 432 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 4 Data size: 432 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint), _col3 (type: bigint) - Stage: Stage-4 + Stage: Stage-2 Dependency Collection Stage: Stage-0 Move Operator tables: - replace: false + replace: true table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.mat2 - Stage: Stage-5 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -406,25 +274,21 @@ STAGE PLANS: Column Types: string, int, bigint, bigint, double Table: default.mat2 - Stage: Stage-6 + Stage: Stage-4 Materialized View Update name: default.mat2 update creation metadata: true PREHOOK: query: alter materialized view mat2 rebuild PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -PREHOOK: Input: default@mat2 PREHOOK: Input: default@tbl_ice PREHOOK: Input: default@tbl_ice_v2 PREHOOK: Output: default@mat2 -PREHOOK: Output: default@mat2 POSTHOOK: query: alter materialized view mat2 rebuild POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -POSTHOOK: Input: default@mat2 POSTHOOK: Input: default@tbl_ice POSTHOOK: Input: default@tbl_ice_v2 POSTHOOK: Output: default@mat2 -POSTHOOK: Output: default@mat2 PREHOOK: query: select * from mat2 PREHOOK: type: QUERY PREHOOK: Input: default@mat2 @@ -433,5 +297,5 @@ POSTHOOK: query: select * from mat2 POSTHOOK: type: QUERY POSTHOOK: Input: default@mat2 POSTHOOK: Output: hdfs://### HDFS PATH ### -five 54 108 2 54.0 -four 53 106 2 53.0 +five 54 216 4 54.0 +four 53 212 4 53.0 diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc6.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc6.q.out index e81eace0ee89..e82bf2d675a3 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc6.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc6.q.out @@ -122,19 +122,17 @@ POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tbl_ice PREHOOK: query: alter materialized view mat1 rebuild PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -PREHOOK: Input: default@mat1 PREHOOK: Input: default@tbl_ice PREHOOK: Input: default@tbl_ice_v2 PREHOOK: Output: default@mat1 POSTHOOK: query: alter materialized view mat1 rebuild POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -POSTHOOK: Input: default@mat1 POSTHOOK: Input: default@tbl_ice POSTHOOK: Input: default@tbl_ice_v2 POSTHOOK: Output: default@mat1 -POSTHOOK: Lineage: mat1._c2 EXPRESSION [(tbl_ice_v2)tbl_ice_v2.FieldSchema(name:f, type:int, comment:null), (mat1)default.mat1.FieldSchema(name:_c2, type:bigint, comment:null), ] -POSTHOOK: Lineage: mat1.b EXPRESSION [(tbl_ice)tbl_ice.FieldSchema(name:b, type:string, comment:null), (mat1)default.mat1.FieldSchema(name:b, type:string, comment:null), ] -POSTHOOK: Lineage: mat1.c EXPRESSION [(tbl_ice)tbl_ice.FieldSchema(name:c, type:int, comment:null), (mat1)default.mat1.FieldSchema(name:c, type:int, comment:null), ] +POSTHOOK: Lineage: mat1._c2 EXPRESSION [(tbl_ice_v2)tbl_ice_v2.FieldSchema(name:f, type:int, comment:null), ] +POSTHOOK: Lineage: mat1.b SIMPLE [(tbl_ice)tbl_ice.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: mat1.c SIMPLE [(tbl_ice)tbl_ice.FieldSchema(name:c, type:int, comment:null), ] PREHOOK: query: select * from mat1 PREHOOK: type: QUERY PREHOOK: Input: default@mat1 diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc7.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc7.q.out index a52e9ae8390f..5df42baca134 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc7.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc7.q.out @@ -62,81 +62,48 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### PREHOOK: query: explain cbo alter materialized view mat1 rebuild PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -PREHOOK: Input: default@mat1 PREHOOK: Input: default@tbl_ice PREHOOK: Output: default@mat1 -PREHOOK: Output: default@mat1 POSTHOOK: query: explain cbo alter materialized view mat1 rebuild POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -POSTHOOK: Input: default@mat1 POSTHOOK: Input: default@tbl_ice POSTHOOK: Output: default@mat1 -POSTHOOK: Output: default@mat1 CBO PLAN: -HiveProject(tbl_ice.a=[$3], _c1=[CASE(IS NULL($1), $4, IS NULL($4), $1, +($4, $1))]) - HiveFilter(condition=[OR($2, IS NULL($2))]) - HiveJoin(condition=[IS NOT DISTINCT FROM($0, $3)], joinType=[right], algorithm=[BucketJoin], cost=[not available]) - HiveProject(a=[$0], _c1=[$1], $f2=[true]) - HiveTableScan(table=[[default, mat1]], table:alias=[default.mat1]) - HiveProject(a=[$0], $f1=[$1]) - HiveAggregate(group=[{0}], agg#0=[count($2)]) - HiveTableScan(table=[[default, tbl_ice]], table:alias=[tbl_ice], fromVersion=[#Masked#]) +HiveProject(tbl_ice.a=[$0], _c1=[$1]) + HiveAggregate(group=[{0}], agg#0=[count($2)]) + HiveTableScan(table=[[default, tbl_ice]], table:alias=[tbl_ice]) PREHOOK: query: explain alter materialized view mat1 rebuild PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -PREHOOK: Input: default@mat1 PREHOOK: Input: default@tbl_ice PREHOOK: Output: default@mat1 -PREHOOK: Output: default@mat1 POSTHOOK: query: explain alter materialized view mat1 rebuild POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -POSTHOOK: Input: default@mat1 POSTHOOK: Input: default@tbl_ice POSTHOOK: Output: default@mat1 -POSTHOOK: Output: default@mat1 STAGE DEPENDENCIES: - Stage-3 is a root stage + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 Stage-4 depends on stages: Stage-3 - Stage-0 depends on stages: Stage-4 - Stage-5 depends on stages: Stage-0 - Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-3 + Stage: Stage-1 Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 - Map Operator Tree: - TableScan - alias: default.mat1 - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), _c1 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: boolean), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) - Execution mode: vectorized - Map 4 Map Operator Tree: TableScan alias: tbl_ice Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: NONE - Version interval from: #Masked# Select Operator expressions: a (type: int), c (type: int) outputColumnNames: a, c @@ -157,74 +124,6 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized Reducer 2 - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - nullSafes: [true] - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 3 Data size: 39 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: _col2 (type: boolean) - Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: bigint) - Filter Operator - predicate: _col2 (type: boolean) - Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col7 (type: int), CASE WHEN (_col1 is null) THEN (_col8) WHEN (_col8 is null) THEN (_col1) ELSE ((_col8 + _col1)) END (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.mat1 - Filter Operator - predicate: _col2 is null (type: boolean) - Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col7 (type: int), CASE WHEN (_col1 is null) THEN (_col8) WHEN (_col8 is null) THEN (_col1) ELSE ((_col8 + _col1)) END (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.mat1 - Reducer 3 - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.mat1 - Reducer 5 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -233,48 +132,45 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) + File Output Operator + compressed: false Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.mat1 - Stage: Stage-4 + Stage: Stage-2 Dependency Collection Stage: Stage-0 Move Operator tables: - replace: false + replace: true table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.mat1 - Stage: Stage-5 + Stage: Stage-3 Stats Work Basic Stats Work: - Stage: Stage-6 + Stage: Stage-4 Materialized View Update name: default.mat1 update creation metadata: true PREHOOK: query: alter materialized view mat1 rebuild PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -PREHOOK: Input: default@mat1 PREHOOK: Input: default@tbl_ice PREHOOK: Output: default@mat1 -PREHOOK: Output: default@mat1 POSTHOOK: query: alter materialized view mat1 rebuild POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -POSTHOOK: Input: default@mat1 POSTHOOK: Input: default@tbl_ice POSTHOOK: Output: default@mat1 -POSTHOOK: Output: default@mat1 PREHOOK: query: select * from mat1 PREHOOK: type: QUERY PREHOOK: Input: default@mat1 diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java index 127cde9a43c8..bd11e3a12938 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java @@ -26,6 +26,8 @@ import org.apache.calcite.plan.hep.HepMatchOrder; import org.apache.calcite.plan.hep.HepProgramBuilder; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelVisitor; +import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMetadataProvider; import org.apache.calcite.rel.metadata.RelMetadataQuery; @@ -314,6 +316,20 @@ protected RelNode applyMaterializedViewRewriting(RelOptPlanner planner, RelNode return calcitePreMVRewritingPlan; } + // For a rebuild, the rewritten plan must still reference at least one source table. + // A trivial view-only rewrite where the entire plan collapses to a scan of the target MV + // is incorrect for a rebuild statement because it would produce + // "INSERT OVERWRITE mv SELECT * FROM mv", dropping any delta accumulated in the source + // tables since the last rebuild. Fall back to the un-rewritten (full-rebuild) plan. + if (getTablesUsed(basePlan).stream().noneMatch(tablesUsedQuery::contains)) { + // MV preparation (HiveAugmentSnapshotMaterializationRule) may have set a + // versionIntervalFrom on the source Table objects to make MV rewriting produce a + // delta-only scan. Since we are falling back to a full rebuild here, clear that + // scan-time filter so the pre-rewriting plan scans all source rows. + clearSourceSnapshotFilters(calcitePreMVRewritingPlan); + return calcitePreMVRewritingPlan; + } + try { if (!HiveMaterializedViewUtils.checkPrivilegeForMaterializedViews(materializedViewsUsedAfterRewrite)) { // if materialized views do not have appropriate privileges, we shouldn't be using them @@ -350,6 +366,19 @@ protected RelNode applyMaterializedViewRewriting(RelOptPlanner planner, RelNode return applyPreJoinOrderingTransforms(basePlan, mdProvider, executorProvider); } + private void clearSourceSnapshotFilters(RelNode plan) { + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + if (node instanceof TableScan) { + Table table = ((RelOptHiveTable) node.getTable()).getHiveTableMD(); + table.setVersionIntervalFrom(null); + } + super.visit(node, ordinal, parent); + } + }.go(plan); + } + private RelNode applyRecordIncrementalRebuildPlan( RelNode basePlan, RelMetadataProvider mdProvider, From 0be0b1e3c70765afa7f6236f424feb8e14a52404 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Mon, 14 Sep 2026 09:07:39 +0100 Subject: [PATCH 22/24] Cleanup Bug.java: CALCITE_7641 and related code (fixed), CALCITE_4200 (unused) --- .../hadoop/hive/ql/optimizer/calcite/Bug.java | 4 - .../rules/views/HiveMaterializedViewRule.java | 182 +++--------------- 2 files changed, 24 insertions(+), 162 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java index 3836f8e05a45..b8e0c58613fc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java @@ -46,8 +46,4 @@ public final class Bug { * CALCITE-6391 is fixed. */ public static final boolean CALCITE_6391_FIXED = false; - /** - * Whether CALCITE-7641 is fixed. - */ - public static final boolean CALCITE_7641_FIXED = false; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewRule.java index db0f338ef1dd..9418d38234d6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewRule.java @@ -18,7 +18,6 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.rules.views; -import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableList; import org.apache.calcite.avatica.util.TimeUnitRange; import org.apache.calcite.plan.RelOptRule; @@ -30,9 +29,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Join; -import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.TableScan; -import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.rules.materialize.MaterializedViewProjectFilterRule; import org.apache.calcite.rel.rules.materialize.MaterializedViewOnlyFilterRule; import org.apache.calcite.rel.rules.materialize.MaterializedViewProjectJoinRule; @@ -40,22 +37,16 @@ import org.apache.calcite.rel.rules.materialize.MaterializedViewProjectAggregateRule; import org.apache.calcite.rel.rules.materialize.MaterializedViewOnlyAggregateRule; import org.apache.calcite.rel.rules.ProjectRemoveRule; -import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexSimplify; -import org.apache.calcite.rex.RexTableInputRef; import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.Util; -import org.apache.hadoop.hive.ql.optimizer.calcite.Bug; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelBuilder; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterProjectTransposeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveJoinProjectTransposeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveProjectMergeRule; -import org.checkerframework.checker.nullness.qual.Nullable; import java.util.List; @@ -86,16 +77,36 @@ public class HiveMaterializedViewRule { .build(); public static final MaterializedViewProjectFilterRule INSTANCE_PROJECT_FILTER = - new HiveMaterializedViewProjectFilterRule(); + (MaterializedViewProjectFilterRule) MaterializedViewProjectFilterRule.Config.DEFAULT + .withGenerateUnionRewriting(true) + .withFastBailOut(false) + .withUnionRewritingPullProgram(PROGRAM) + .withRelBuilderFactory(HiveRelFactories.HIVE_BUILDER) + .toRule(); public static final MaterializedViewOnlyFilterRule INSTANCE_FILTER = - new HiveMaterializedViewOnlyFilterRule(); + (MaterializedViewOnlyFilterRule) MaterializedViewOnlyFilterRule.Config.DEFAULT + .withGenerateUnionRewriting(true) + .withFastBailOut(false) + .withUnionRewritingPullProgram(PROGRAM) + .withRelBuilderFactory(HiveRelFactories.HIVE_BUILDER) + .toRule(); public static final MaterializedViewProjectJoinRule INSTANCE_PROJECT_JOIN = - new HiveMaterializedViewProjectJoinRule(); + (MaterializedViewProjectJoinRule) MaterializedViewProjectJoinRule.Config.DEFAULT + .withGenerateUnionRewriting(true) + .withFastBailOut(false) + .withUnionRewritingPullProgram(PROGRAM) + .withRelBuilderFactory(HiveRelFactories.HIVE_BUILDER) + .toRule(); public static final MaterializedViewOnlyJoinRule INSTANCE_JOIN = - new HiveMaterializedViewOnlyJoinRule(); + (MaterializedViewOnlyJoinRule) MaterializedViewOnlyJoinRule.Config.DEFAULT + .withGenerateUnionRewriting(true) + .withFastBailOut(false) + .withUnionRewritingPullProgram(PROGRAM) + .withRelBuilderFactory(HiveRelFactories.HIVE_BUILDER) + .toRule(); public static final HiveMaterializedViewProjectAggregateRule INSTANCE_PROJECT_AGGREGATE = new HiveMaterializedViewProjectAggregateRule(); @@ -118,26 +129,6 @@ public HiveMaterializedViewProjectAggregateRule() { super(HiveRelFactories.HIVE_BUILDER, true, PROGRAM); } - // Overridden to avoid CALCITE-7641 - @Override protected @Nullable RelNode rewriteQuery( - RelBuilder relBuilder, - RexBuilder rexBuilder, - RexSimplify simplify, - RelMetadataQuery mq, - RexNode compensationColumnsEquiPred, - RexNode otherCompensationPred, - @Nullable Project topProject, - RelNode node, - BiMap queryToViewTableMapping, - EquivalenceClasses viewEC, EquivalenceClasses queryEC) { - if (Bug.CALCITE_7641_FIXED) { - throw new AssertionError("Overriding should be removed"); - } - RelNode unwrappedNode = HiveCalciteUtil.stripHepVertices(node); - return super.rewriteQuery(relBuilder, rexBuilder, simplify, mq, compensationColumnsEquiPred, - otherCompensationPred, topProject, unwrappedNode, queryToViewTableMapping, viewEC, queryEC); - } - @Override protected SqlFunction getFloorSqlFunction(TimeUnitRange flag) { return HiveRelBuilder.getFloorSqlFunction(flag); @@ -149,137 +140,12 @@ public HiveMaterializedViewOnlyAggregateRule() { super(HiveRelFactories.HIVE_BUILDER, true, PROGRAM); } - // Overridden to avoid CALCITE-7641 - @Override protected @Nullable RelNode rewriteQuery( - RelBuilder relBuilder, - RexBuilder rexBuilder, - RexSimplify simplify, - RelMetadataQuery mq, - RexNode compensationColumnsEquiPred, - RexNode otherCompensationPred, - @Nullable Project topProject, - RelNode node, - BiMap queryToViewTableMapping, - EquivalenceClasses viewEC, EquivalenceClasses queryEC) { - if (Bug.CALCITE_7641_FIXED) { - throw new AssertionError("Overriding should be removed"); - } - RelNode unwrappedNode = HiveCalciteUtil.stripHepVertices(node); - return super.rewriteQuery(relBuilder, rexBuilder, simplify, mq, compensationColumnsEquiPred, - otherCompensationPred, topProject, unwrappedNode, queryToViewTableMapping, viewEC, queryEC); - } - @Override protected SqlFunction getFloorSqlFunction(TimeUnitRange flag) { return HiveRelBuilder.getFloorSqlFunction(flag); } } - protected static class HiveMaterializedViewProjectFilterRule extends MaterializedViewProjectFilterRule { - public HiveMaterializedViewProjectFilterRule() { - super(HiveRelFactories.HIVE_BUILDER, true, PROGRAM, false); - } - - // Overridden to avoid CALCITE-7641 - @Override protected @Nullable RelNode rewriteQuery( - RelBuilder relBuilder, - RexBuilder rexBuilder, - RexSimplify simplify, - RelMetadataQuery mq, - RexNode compensationColumnsEquiPred, - RexNode otherCompensationPred, - @Nullable Project topProject, - RelNode node, - BiMap viewToQueryTableMapping, - EquivalenceClasses viewEC, EquivalenceClasses queryEC) { - if (Bug.CALCITE_7641_FIXED) { - throw new AssertionError("Overriding should be removed"); - } - RelNode unwrappedNode = HiveCalciteUtil.stripHepVertices(node); - return super.rewriteQuery(relBuilder, rexBuilder, simplify, mq, compensationColumnsEquiPred, - otherCompensationPred, topProject, unwrappedNode, viewToQueryTableMapping, viewEC, queryEC); - } - } - - protected static class HiveMaterializedViewOnlyFilterRule extends MaterializedViewOnlyFilterRule { - public HiveMaterializedViewOnlyFilterRule() { - super(HiveRelFactories.HIVE_BUILDER, true, PROGRAM, false); - } - - // Overridden to avoid CALCITE-7641 - @Override protected @Nullable RelNode rewriteQuery( - RelBuilder relBuilder, - RexBuilder rexBuilder, - RexSimplify simplify, - RelMetadataQuery mq, - RexNode compensationColumnsEquiPred, - RexNode otherCompensationPred, - @Nullable Project topProject, - RelNode node, - BiMap viewToQueryTableMapping, - EquivalenceClasses viewEC, EquivalenceClasses queryEC) { - if (Bug.CALCITE_7641_FIXED) { - throw new AssertionError("Overriding should be removed"); - } - RelNode unwrappedNode = HiveCalciteUtil.stripHepVertices(node); - return super.rewriteQuery(relBuilder, rexBuilder, simplify, mq, compensationColumnsEquiPred, - otherCompensationPred, topProject, unwrappedNode, viewToQueryTableMapping, viewEC, queryEC); - } - } - - protected static class HiveMaterializedViewProjectJoinRule extends MaterializedViewProjectJoinRule { - public HiveMaterializedViewProjectJoinRule() { - super(HiveRelFactories.HIVE_BUILDER, true, PROGRAM, false); - } - - // Overridden to avoid CALCITE-7641 - @Override protected @Nullable RelNode rewriteQuery( - RelBuilder relBuilder, - RexBuilder rexBuilder, - RexSimplify simplify, - RelMetadataQuery mq, - RexNode compensationColumnsEquiPred, - RexNode otherCompensationPred, - @Nullable Project topProject, - RelNode node, - BiMap viewToQueryTableMapping, - EquivalenceClasses viewEC, EquivalenceClasses queryEC) { - if (Bug.CALCITE_7641_FIXED) { - throw new AssertionError("Overriding should be removed"); - } - RelNode unwrappedNode = HiveCalciteUtil.stripHepVertices(node); - return super.rewriteQuery(relBuilder, rexBuilder, simplify, mq, compensationColumnsEquiPred, - otherCompensationPred, topProject, unwrappedNode, viewToQueryTableMapping, viewEC, queryEC); - } - } - - protected static class HiveMaterializedViewOnlyJoinRule extends MaterializedViewOnlyJoinRule { - public HiveMaterializedViewOnlyJoinRule() { - super(HiveRelFactories.HIVE_BUILDER, true, PROGRAM, false); - } - - // Overridden to avoid CALCITE-7641 - @Override protected @Nullable RelNode rewriteQuery( - RelBuilder relBuilder, - RexBuilder rexBuilder, - RexSimplify simplify, - RelMetadataQuery mq, - RexNode compensationColumnsEquiPred, - RexNode otherCompensationPred, - @Nullable Project topProject, - RelNode node, - BiMap viewToQueryTableMapping, - EquivalenceClasses viewEC, - EquivalenceClasses queryEC) { - if (Bug.CALCITE_7641_FIXED) { - throw new AssertionError("Overriding should be removed"); - } - RelNode unwrappedNode = HiveCalciteUtil.stripHepVertices(node); - return super.rewriteQuery(relBuilder, rexBuilder, simplify, mq, compensationColumnsEquiPred, - otherCompensationPred, topProject, unwrappedNode, viewToQueryTableMapping, viewEC, queryEC); - } - } - /** * This rule is used within the PROGRAM that rewrites the query for * partial rewritings. Its goal is to extract the RelNode from the From a088043e5bc9f7dfc0601d424f789c04f595bb34 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Fri, 25 Sep 2026 11:42:59 +0100 Subject: [PATCH 23/24] Alteration in query64 plans due to CALCITE-7687 (metadata selectivity bugfix on Aggregate) --- .../perf/tpcds30tb/cte/cbo_query64.q.out | 24 +- .../perf/tpcds30tb/json/query64.q.out | 270 ++++---- .../perf/tpcds30tb/tez/cbo_query64.q.out | 24 +- .../perf/tpcds30tb/tez/query64.q.out | 630 +++++++++--------- 4 files changed, 474 insertions(+), 474 deletions(-) diff --git a/ql/src/test/results/clientpositive/perf/tpcds30tb/cte/cbo_query64.q.out b/ql/src/test/results/clientpositive/perf/tpcds30tb/cte/cbo_query64.q.out index 3e284272e21c..ca0a23f6a8d9 100644 --- a/ql/src/test/results/clientpositive/perf/tpcds30tb/cte/cbo_query64.q.out +++ b/ql/src/test/results/clientpositive/perf/tpcds30tb/cte/cbo_query64.q.out @@ -40,12 +40,12 @@ HiveProject(s_store_sk=[$0], s_store_name=[$5], s_zip=[$25]) CBO PLAN: HiveProject(cs1.product_name=[$0], cs1.store_name=[$1], cs1.store_zip=[$2], cs1.b_street_number=[$3], cs1.b_streen_name=[$4], cs1.b_city=[$5], cs1.b_zip=[$6], cs1.c_street_number=[$7], cs1.c_street_name=[$8], cs1.c_city=[$9], cs1.c_zip=[$10], cs1.syear=[CAST(2000):INTEGER], cs1.cnt=[$11], cs1.s1=[$12], cs1.s2=[$13], cs1.s3=[$14], cs2.s1=[$15], cs2.s2=[$16], cs2.s3=[$17], cs2.syear=[CAST(2001):INTEGER], cs2.cnt=[$18]) HiveSortLimit(sort0=[$0], sort1=[$1], sort2=[$18], dir0=[ASC], dir1=[ASC], dir2=[ASC]) - HiveProject(product_name=[$7], store_name=[$9], store_zip=[$10], b_street_number=[$11], b_streen_name=[$12], b_city=[$13], b_zip=[$14], c_street_number=[$15], c_street_name=[$16], c_city=[$17], c_zip=[$18], cnt=[$19], s1=[$20], s2=[$21], s3=[$22], s11=[$4], s21=[$5], s31=[$6], cnt1=[$3]) - HiveJoin(condition=[AND(=($9, $1), =($8, $0), =($10, $2), <=($3, $19))], joinType=[inner], algorithm=[none], cost=[not available]) - HiveProject($f1=[$0], $f2=[$1], $f3=[$2], $f15=[$13], $f16=[$14], $f17=[$15], $f18=[$16]) - HiveFilter(condition=[IS NOT NULL($13)]) - HiveProject(i_item_sk=[$4], s_store_name=[$5], s_zip=[$6], ca_street_number=[$0], ca_street_name=[$1], ca_city=[$2], ca_zip=[$3], ca_street_number0=[$9], ca_street_name0=[$10], ca_city0=[$11], ca_zip0=[$12], d_year=[$7], d_year0=[$8], $f13=[$13], $f14=[$14], $f15=[$15], $f16=[$16]) - HiveAggregate(group=[{3, 4, 5, 6, 24, 29, 30, 34, 36, 42, 43, 44, 45}], agg#0=[count()], agg#1=[sum($20)], agg#2=[sum($21)], agg#3=[sum($22)]) + HiveProject(product_name=[$0], store_name=[$2], store_zip=[$3], b_street_number=[$4], b_streen_name=[$5], b_city=[$6], b_zip=[$7], c_street_number=[$8], c_street_name=[$9], c_city=[$10], c_zip=[$11], cnt=[$12], s1=[$13], s2=[$14], s3=[$15], s11=[$20], s21=[$21], s31=[$22], cnt1=[$19]) + HiveJoin(condition=[AND(=($2, $17), =($1, $16), =($3, $18), <=($19, $12))], joinType=[inner], algorithm=[none], cost=[not available]) + HiveProject($f0=[$0], $f1=[$1], $f2=[$2], $f3=[$3], $f4=[$4], $f5=[$5], $f6=[$6], $f7=[$7], $f8=[$8], $f9=[$9], $f10=[$10], $f11=[$11], $f15=[$14], $f16=[$15], $f17=[$16], $f18=[$17]) + HiveFilter(condition=[IS NOT NULL($14)]) + HiveProject(i_product_name=[$5], i_item_sk=[$4], s_store_name=[$6], s_zip=[$7], ca_street_number=[$0], ca_street_name=[$1], ca_city=[$2], ca_zip=[$3], ca_street_number0=[$10], ca_street_name0=[$11], ca_city0=[$12], ca_zip0=[$13], d_year=[$8], d_year0=[$9], $f14=[$14], $f15=[$15], $f16=[$16], $f17=[$17]) + HiveAggregate(group=[{3, 4, 5, 6, 24, 25, 29, 30, 34, 36, 42, 43, 44, 45}], agg#0=[count()], agg#1=[sum($20)], agg#2=[sum($21)], agg#3=[sum($22)]) HiveJoin(condition=[=($10, $41)], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[AND(=($8, $39), <>($38, $40))], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($15, $37)], joinType=[inner], algorithm=[none], cost=[not available]) @@ -76,7 +76,7 @@ HiveProject(cs1.product_name=[$0], cs1.store_name=[$1], cs1.store_zip=[$2], cs1. HiveFilter(condition=[AND(<=(36.00:DECIMAL(12, 2), $5), <=($5, 45.00:DECIMAL(12, 2)), OR(=($17, 'burnished '), =($17, 'chocolate '), =($17, 'dim '), =($17, 'maroon '), =($17, 'navajo '), =($17, 'steel ')))]) HiveTableScan(table=[[default, item]], table:alias=[item]) HiveProject(d_date_sk=[$0]) - HiveFilter(condition=[=($6, 2001)]) + HiveFilter(condition=[=($6, 2000)]) HiveTableScan(table=[[default, date_dim]], table:alias=[d1]) HiveProject(cs_item_sk=[$0]) HiveFilter(condition=[>($1, *(2:DECIMAL(10, 0), $2))]) @@ -105,10 +105,10 @@ HiveProject(cs1.product_name=[$0], cs1.store_name=[$1], cs1.store_zip=[$2], cs1. HiveTableScan(table=[[default, customer_demographics]], table:alias=[cd2]) HiveProject(ca_address_sk=[$0], ca_street_number=[$2], ca_street_name=[$3], ca_city=[$6], ca_zip=[$9]) HiveTableScan(table=[[default, customer_address]], table:alias=[ad2]) - HiveProject($f0=[$0], $f1=[$1], $f2=[$2], $f3=[$3], $f4=[$4], $f5=[$5], $f6=[$6], $f7=[$7], $f8=[$8], $f9=[$9], $f10=[$10], $f11=[$11], $f15=[$14], $f16=[$15], $f17=[$16], $f18=[$17]) - HiveFilter(condition=[IS NOT NULL($14)]) - HiveProject(i_product_name=[$5], i_item_sk=[$4], s_store_name=[$6], s_zip=[$7], ca_street_number=[$0], ca_street_name=[$1], ca_city=[$2], ca_zip=[$3], ca_street_number0=[$10], ca_street_name0=[$11], ca_city0=[$12], ca_zip0=[$13], d_year=[$8], d_year0=[$9], $f14=[$14], $f15=[$15], $f16=[$16], $f17=[$17]) - HiveAggregate(group=[{3, 4, 5, 6, 24, 25, 29, 30, 34, 36, 42, 43, 44, 45}], agg#0=[count()], agg#1=[sum($20)], agg#2=[sum($21)], agg#3=[sum($22)]) + HiveProject($f1=[$0], $f2=[$1], $f3=[$2], $f15=[$13], $f16=[$14], $f17=[$15], $f18=[$16]) + HiveFilter(condition=[IS NOT NULL($13)]) + HiveProject(i_item_sk=[$4], s_store_name=[$5], s_zip=[$6], ca_street_number=[$0], ca_street_name=[$1], ca_city=[$2], ca_zip=[$3], ca_street_number0=[$9], ca_street_name0=[$10], ca_city0=[$11], ca_zip0=[$12], d_year=[$7], d_year0=[$8], $f13=[$13], $f14=[$14], $f15=[$15], $f16=[$16]) + HiveAggregate(group=[{3, 4, 5, 6, 24, 29, 30, 34, 36, 42, 43, 44, 45}], agg#0=[count()], agg#1=[sum($20)], agg#2=[sum($21)], agg#3=[sum($22)]) HiveJoin(condition=[=($10, $41)], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[AND(=($8, $39), <>($38, $40))], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($15, $37)], joinType=[inner], algorithm=[none], cost=[not available]) @@ -136,7 +136,7 @@ HiveProject(cs1.product_name=[$0], cs1.store_name=[$1], cs1.store_zip=[$2], cs1. HiveFilter(condition=[AND(<=(36.00:DECIMAL(12, 2), $5), <=($5, 45.00:DECIMAL(12, 2)), OR(=($17, 'burnished '), =($17, 'chocolate '), =($17, 'dim '), =($17, 'maroon '), =($17, 'navajo '), =($17, 'steel ')))]) HiveTableScan(table=[[default, item]], table:alias=[item]) HiveProject(d_date_sk=[$0]) - HiveFilter(condition=[=($6, 2000)]) + HiveFilter(condition=[=($6, 2001)]) HiveTableScan(table=[[default, date_dim]], table:alias=[d1]) HiveProject(cs_item_sk=[$0]) HiveFilter(condition=[>($1, *(2:DECIMAL(10, 0), $2))]) diff --git a/ql/src/test/results/clientpositive/perf/tpcds30tb/json/query64.q.out b/ql/src/test/results/clientpositive/perf/tpcds30tb/json/query64.q.out index 8bda4c5d0557..eed5ab36f920 100644 --- a/ql/src/test/results/clientpositive/perf/tpcds30tb/json/query64.q.out +++ b/ql/src/test/results/clientpositive/perf/tpcds30tb/json/query64.q.out @@ -2229,7 +2229,7 @@ "name": "$6" }, { - "literal": 2001, + "literal": 2000, "type": { "type": "INTEGER", "nullable": false @@ -5774,6 +5774,7 @@ 5, 6, 24, + 25, 29, 30, 34, @@ -5859,6 +5860,7 @@ "id": "58", "relOp": "org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject", "fields": [ + "i_product_name", "i_item_sk", "s_store_name", "s_zip", @@ -5872,24 +5874,28 @@ "ca_zip0", "d_year", "d_year0", - "$f13", "$f14", "$f15", - "$f16" + "$f16", + "$f17" ], "exprs": [ - { - "input": 4, - "name": "$4" - }, { "input": 5, "name": "$5" }, + { + "input": 4, + "name": "$4" + }, { "input": 6, "name": "$6" }, + { + "input": 7, + "name": "$7" + }, { "input": 0, "name": "$0" @@ -5906,10 +5912,6 @@ "input": 3, "name": "$3" }, - { - "input": 9, - "name": "$9" - }, { "input": 10, "name": "$10" @@ -5923,16 +5925,16 @@ "name": "$12" }, { - "input": 7, - "name": "$7" + "input": 13, + "name": "$13" }, { "input": 8, "name": "$8" }, { - "input": 13, - "name": "$13" + "input": 9, + "name": "$9" }, { "input": 14, @@ -5945,6 +5947,10 @@ { "input": 16, "name": "$16" + }, + { + "input": 17, + "name": "$17" } ], "rowCount": 4.277045496443079E88 @@ -5960,8 +5966,8 @@ }, "operands": [ { - "input": 13, - "name": "$13" + "input": 14, + "name": "$14" } ] }, @@ -5971,9 +5977,18 @@ "id": "60", "relOp": "org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject", "fields": [ + "$f0", "$f1", "$f2", "$f3", + "$f4", + "$f5", + "$f6", + "$f7", + "$f8", + "$f9", + "$f10", + "$f11", "$f15", "$f16", "$f17", @@ -5993,8 +6008,40 @@ "name": "$2" }, { - "input": 13, - "name": "$13" + "input": 3, + "name": "$3" + }, + { + "input": 4, + "name": "$4" + }, + { + "input": 5, + "name": "$5" + }, + { + "input": 6, + "name": "$6" + }, + { + "input": 7, + "name": "$7" + }, + { + "input": 8, + "name": "$8" + }, + { + "input": 9, + "name": "$9" + }, + { + "input": 10, + "name": "$10" + }, + { + "input": 11, + "name": "$11" }, { "input": 14, @@ -6007,6 +6054,10 @@ { "input": 16, "name": "$16" + }, + { + "input": 17, + "name": "$17" } ], "rowCount": 3.849340946798771E88 @@ -6542,7 +6593,7 @@ "name": "$6" }, { - "literal": 2000, + "literal": 2001, "type": { "type": "INTEGER", "nullable": false @@ -7504,7 +7555,6 @@ 5, 6, 24, - 25, 29, 30, 34, @@ -7590,7 +7640,6 @@ "id": "103", "relOp": "org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject", "fields": [ - "i_product_name", "i_item_sk", "s_store_name", "s_zip", @@ -7604,27 +7653,23 @@ "ca_zip0", "d_year", "d_year0", + "$f13", "$f14", "$f15", - "$f16", - "$f17" + "$f16" ], "exprs": [ - { - "input": 5, - "name": "$5" - }, { "input": 4, "name": "$4" }, { - "input": 6, - "name": "$6" + "input": 5, + "name": "$5" }, { - "input": 7, - "name": "$7" + "input": 6, + "name": "$6" }, { "input": 0, @@ -7642,6 +7687,10 @@ "input": 3, "name": "$3" }, + { + "input": 9, + "name": "$9" + }, { "input": 10, "name": "$10" @@ -7655,16 +7704,16 @@ "name": "$12" }, { - "input": 13, - "name": "$13" + "input": 7, + "name": "$7" }, { "input": 8, "name": "$8" }, { - "input": 9, - "name": "$9" + "input": 13, + "name": "$13" }, { "input": 14, @@ -7677,10 +7726,6 @@ { "input": 16, "name": "$16" - }, - { - "input": 17, - "name": "$17" } ], "rowCount": 4.277045496443079E88 @@ -7696,8 +7741,8 @@ }, "operands": [ { - "input": 14, - "name": "$14" + "input": 13, + "name": "$13" } ] }, @@ -7707,18 +7752,9 @@ "id": "105", "relOp": "org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject", "fields": [ - "$f0", "$f1", "$f2", "$f3", - "$f4", - "$f5", - "$f6", - "$f7", - "$f8", - "$f9", - "$f10", - "$f11", "$f15", "$f16", "$f17", @@ -7738,40 +7774,8 @@ "name": "$2" }, { - "input": 3, - "name": "$3" - }, - { - "input": 4, - "name": "$4" - }, - { - "input": 5, - "name": "$5" - }, - { - "input": 6, - "name": "$6" - }, - { - "input": 7, - "name": "$7" - }, - { - "input": 8, - "name": "$8" - }, - { - "input": 9, - "name": "$9" - }, - { - "input": 10, - "name": "$10" - }, - { - "input": 11, - "name": "$11" + "input": 13, + "name": "$13" }, { "input": 14, @@ -7784,10 +7788,6 @@ { "input": 16, "name": "$16" - }, - { - "input": 17, - "name": "$17" } ], "rowCount": 3.849340946798771E88 @@ -7810,12 +7810,12 @@ }, "operands": [ { - "input": 9, - "name": "$9" + "input": 2, + "name": "$2" }, { - "input": 1, - "name": "$1" + "input": 17, + "name": "$17" } ] }, @@ -7827,12 +7827,12 @@ }, "operands": [ { - "input": 8, - "name": "$8" + "input": 1, + "name": "$1" }, { - "input": 0, - "name": "$0" + "input": 16, + "name": "$16" } ] }, @@ -7844,12 +7844,12 @@ }, "operands": [ { - "input": 10, - "name": "$10" + "input": 3, + "name": "$3" }, { - "input": 2, - "name": "$2" + "input": 18, + "name": "$18" } ] }, @@ -7860,13 +7860,13 @@ "syntax": "BINARY" }, "operands": [ - { - "input": 3, - "name": "$3" - }, { "input": 19, "name": "$19" + }, + { + "input": 12, + "name": "$12" } ] } @@ -7906,10 +7906,38 @@ "cnt1" ], "exprs": [ + { + "input": 0, + "name": "$0" + }, + { + "input": 2, + "name": "$2" + }, + { + "input": 3, + "name": "$3" + }, + { + "input": 4, + "name": "$4" + }, + { + "input": 5, + "name": "$5" + }, + { + "input": 6, + "name": "$6" + }, { "input": 7, "name": "$7" }, + { + "input": 8, + "name": "$8" + }, { "input": 9, "name": "$9" @@ -7938,22 +7966,6 @@ "input": 15, "name": "$15" }, - { - "input": 16, - "name": "$16" - }, - { - "input": 17, - "name": "$17" - }, - { - "input": 18, - "name": "$18" - }, - { - "input": 19, - "name": "$19" - }, { "input": 20, "name": "$20" @@ -7967,20 +7979,8 @@ "name": "$22" }, { - "input": 4, - "name": "$4" - }, - { - "input": 5, - "name": "$5" - }, - { - "input": 6, - "name": "$6" - }, - { - "input": 3, - "name": "$3" + "input": 19, + "name": "$19" } ], "rowCount": 2.500440591043405E174 diff --git a/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/cbo_query64.q.out b/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/cbo_query64.q.out index ee9867c5b84f..205a425770e4 100644 --- a/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/cbo_query64.q.out +++ b/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/cbo_query64.q.out @@ -1,12 +1,12 @@ CBO PLAN: HiveProject(cs1.product_name=[$0], cs1.store_name=[$1], cs1.store_zip=[$2], cs1.b_street_number=[$3], cs1.b_streen_name=[$4], cs1.b_city=[$5], cs1.b_zip=[$6], cs1.c_street_number=[$7], cs1.c_street_name=[$8], cs1.c_city=[$9], cs1.c_zip=[$10], cs1.syear=[CAST(2000):INTEGER], cs1.cnt=[$11], cs1.s1=[$12], cs1.s2=[$13], cs1.s3=[$14], cs2.s1=[$15], cs2.s2=[$16], cs2.s3=[$17], cs2.syear=[CAST(2001):INTEGER], cs2.cnt=[$18]) HiveSortLimit(sort0=[$0], sort1=[$1], sort2=[$18], dir0=[ASC], dir1=[ASC], dir2=[ASC]) - HiveProject(product_name=[$7], store_name=[$9], store_zip=[$10], b_street_number=[$11], b_streen_name=[$12], b_city=[$13], b_zip=[$14], c_street_number=[$15], c_street_name=[$16], c_city=[$17], c_zip=[$18], cnt=[$19], s1=[$20], s2=[$21], s3=[$22], s11=[$4], s21=[$5], s31=[$6], cnt1=[$3]) - HiveJoin(condition=[AND(=($9, $1), =($8, $0), =($10, $2), <=($3, $19))], joinType=[inner], algorithm=[none], cost=[not available]) - HiveProject($f1=[$0], $f2=[$1], $f3=[$2], $f15=[$13], $f16=[$14], $f17=[$15], $f18=[$16]) - HiveFilter(condition=[IS NOT NULL($13)]) - HiveProject(i_item_sk=[$4], s_store_name=[$5], s_zip=[$6], ca_street_number=[$0], ca_street_name=[$1], ca_city=[$2], ca_zip=[$3], ca_street_number0=[$9], ca_street_name0=[$10], ca_city0=[$11], ca_zip0=[$12], d_year=[$7], d_year0=[$8], $f13=[$13], $f14=[$14], $f15=[$15], $f16=[$16]) - HiveAggregate(group=[{3, 4, 5, 6, 24, 29, 30, 34, 36, 42, 43, 44, 45}], agg#0=[count()], agg#1=[sum($20)], agg#2=[sum($21)], agg#3=[sum($22)]) + HiveProject(product_name=[$0], store_name=[$2], store_zip=[$3], b_street_number=[$4], b_streen_name=[$5], b_city=[$6], b_zip=[$7], c_street_number=[$8], c_street_name=[$9], c_city=[$10], c_zip=[$11], cnt=[$12], s1=[$13], s2=[$14], s3=[$15], s11=[$20], s21=[$21], s31=[$22], cnt1=[$19]) + HiveJoin(condition=[AND(=($2, $17), =($1, $16), =($3, $18), <=($19, $12))], joinType=[inner], algorithm=[none], cost=[not available]) + HiveProject($f0=[$0], $f1=[$1], $f2=[$2], $f3=[$3], $f4=[$4], $f5=[$5], $f6=[$6], $f7=[$7], $f8=[$8], $f9=[$9], $f10=[$10], $f11=[$11], $f15=[$14], $f16=[$15], $f17=[$16], $f18=[$17]) + HiveFilter(condition=[IS NOT NULL($14)]) + HiveProject(i_product_name=[$5], i_item_sk=[$4], s_store_name=[$6], s_zip=[$7], ca_street_number=[$0], ca_street_name=[$1], ca_city=[$2], ca_zip=[$3], ca_street_number0=[$10], ca_street_name0=[$11], ca_city0=[$12], ca_zip0=[$13], d_year=[$8], d_year0=[$9], $f14=[$14], $f15=[$15], $f16=[$16], $f17=[$17]) + HiveAggregate(group=[{3, 4, 5, 6, 24, 25, 29, 30, 34, 36, 42, 43, 44, 45}], agg#0=[count()], agg#1=[sum($20)], agg#2=[sum($21)], agg#3=[sum($22)]) HiveJoin(condition=[=($10, $41)], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[AND(=($8, $39), <>($38, $40))], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($15, $37)], joinType=[inner], algorithm=[none], cost=[not available]) @@ -35,7 +35,7 @@ HiveProject(cs1.product_name=[$0], cs1.store_name=[$1], cs1.store_zip=[$2], cs1. HiveFilter(condition=[AND(BETWEEN(false, $5, 36.00:DECIMAL(12, 2), 45.00:DECIMAL(12, 2)), IN($17, 'burnished ', 'chocolate ', 'dim ', 'maroon ', 'navajo ', 'steel '))]) HiveTableScan(table=[[default, item]], table:alias=[item]) HiveProject(d_date_sk=[$0]) - HiveFilter(condition=[=($6, 2001)]) + HiveFilter(condition=[=($6, 2000)]) HiveTableScan(table=[[default, date_dim]], table:alias=[d1]) HiveProject(cs_item_sk=[$0]) HiveFilter(condition=[>($1, *(2:DECIMAL(10, 0), $2))]) @@ -64,10 +64,10 @@ HiveProject(cs1.product_name=[$0], cs1.store_name=[$1], cs1.store_zip=[$2], cs1. HiveTableScan(table=[[default, customer_demographics]], table:alias=[cd2]) HiveProject(ca_address_sk=[$0], ca_street_number=[$2], ca_street_name=[$3], ca_city=[$6], ca_zip=[$9]) HiveTableScan(table=[[default, customer_address]], table:alias=[ad2]) - HiveProject($f0=[$0], $f1=[$1], $f2=[$2], $f3=[$3], $f4=[$4], $f5=[$5], $f6=[$6], $f7=[$7], $f8=[$8], $f9=[$9], $f10=[$10], $f11=[$11], $f15=[$14], $f16=[$15], $f17=[$16], $f18=[$17]) - HiveFilter(condition=[IS NOT NULL($14)]) - HiveProject(i_product_name=[$5], i_item_sk=[$4], s_store_name=[$6], s_zip=[$7], ca_street_number=[$0], ca_street_name=[$1], ca_city=[$2], ca_zip=[$3], ca_street_number0=[$10], ca_street_name0=[$11], ca_city0=[$12], ca_zip0=[$13], d_year=[$8], d_year0=[$9], $f14=[$14], $f15=[$15], $f16=[$16], $f17=[$17]) - HiveAggregate(group=[{3, 4, 5, 6, 24, 25, 29, 30, 34, 36, 42, 43, 44, 45}], agg#0=[count()], agg#1=[sum($20)], agg#2=[sum($21)], agg#3=[sum($22)]) + HiveProject($f1=[$0], $f2=[$1], $f3=[$2], $f15=[$13], $f16=[$14], $f17=[$15], $f18=[$16]) + HiveFilter(condition=[IS NOT NULL($13)]) + HiveProject(i_item_sk=[$4], s_store_name=[$5], s_zip=[$6], ca_street_number=[$0], ca_street_name=[$1], ca_city=[$2], ca_zip=[$3], ca_street_number0=[$9], ca_street_name0=[$10], ca_city0=[$11], ca_zip0=[$12], d_year=[$7], d_year0=[$8], $f13=[$13], $f14=[$14], $f15=[$15], $f16=[$16]) + HiveAggregate(group=[{3, 4, 5, 6, 24, 29, 30, 34, 36, 42, 43, 44, 45}], agg#0=[count()], agg#1=[sum($20)], agg#2=[sum($21)], agg#3=[sum($22)]) HiveJoin(condition=[=($10, $41)], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[AND(=($8, $39), <>($38, $40))], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($15, $37)], joinType=[inner], algorithm=[none], cost=[not available]) @@ -96,7 +96,7 @@ HiveProject(cs1.product_name=[$0], cs1.store_name=[$1], cs1.store_zip=[$2], cs1. HiveFilter(condition=[AND(BETWEEN(false, $5, 36.00:DECIMAL(12, 2), 45.00:DECIMAL(12, 2)), IN($17, 'burnished ', 'chocolate ', 'dim ', 'maroon ', 'navajo ', 'steel '))]) HiveTableScan(table=[[default, item]], table:alias=[item]) HiveProject(d_date_sk=[$0]) - HiveFilter(condition=[=($6, 2000)]) + HiveFilter(condition=[=($6, 2001)]) HiveTableScan(table=[[default, date_dim]], table:alias=[d1]) HiveProject(cs_item_sk=[$0]) HiveFilter(condition=[>($1, *(2:DECIMAL(10, 0), $2))]) diff --git a/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/query64.q.out b/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/query64.q.out index d38476f52ca9..74b46ffb204b 100644 --- a/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/query64.q.out +++ b/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/query64.q.out @@ -7,34 +7,34 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 19 (BROADCAST_EDGE), Map 31 (BROADCAST_EDGE), Reducer 9 (BROADCAST_EDGE) - Map 13 <- Reducer 32 (BROADCAST_EDGE), Reducer 33 (BROADCAST_EDGE) - Map 25 <- Map 19 (BROADCAST_EDGE), Map 31 (BROADCAST_EDGE), Reducer 12 (BROADCAST_EDGE), Reducer 5 (BROADCAST_EDGE) - Map 35 <- Reducer 12 (BROADCAST_EDGE), Reducer 32 (BROADCAST_EDGE), Reducer 33 (BROADCAST_EDGE), Reducer 9 (BROADCAST_EDGE) - Map 6 <- Reducer 32 (BROADCAST_EDGE), Reducer 33 (BROADCAST_EDGE) - Reducer 10 <- Map 13 (CUSTOM_SIMPLE_EDGE), Map 6 (CUSTOM_SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (CUSTOM_SIMPLE_EDGE) - Reducer 17 <- Map 16 (SIMPLE_EDGE) - Reducer 18 <- Map 16 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 14 (BROADCAST_EDGE), Map 34 (CUSTOM_SIMPLE_EDGE), Reducer 8 (BROADCAST_EDGE) - Reducer 20 <- Map 19 (SIMPLE_EDGE) - Reducer 21 <- Map 19 (SIMPLE_EDGE) - Reducer 23 <- Map 22 (SIMPLE_EDGE) - Reducer 24 <- Map 22 (SIMPLE_EDGE) - Reducer 26 <- Map 14 (BROADCAST_EDGE), Map 25 (CUSTOM_SIMPLE_EDGE), Map 34 (CUSTOM_SIMPLE_EDGE), Reducer 11 (BROADCAST_EDGE) - Reducer 27 <- Map 14 (BROADCAST_EDGE), Map 15 (BROADCAST_EDGE), Map 16 (BROADCAST_EDGE), Map 19 (BROADCAST_EDGE), Map 22 (BROADCAST_EDGE), Map 35 (CUSTOM_SIMPLE_EDGE), Reducer 18 (BROADCAST_EDGE), Reducer 21 (BROADCAST_EDGE), Reducer 24 (BROADCAST_EDGE), Reducer 26 (CUSTOM_SIMPLE_EDGE) - Reducer 28 <- Reducer 27 (SIMPLE_EDGE) - Reducer 29 <- Reducer 28 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Map 14 (BROADCAST_EDGE), Map 15 (BROADCAST_EDGE), Map 16 (BROADCAST_EDGE), Map 19 (BROADCAST_EDGE), Map 22 (BROADCAST_EDGE), Map 35 (CUSTOM_SIMPLE_EDGE), Reducer 17 (BROADCAST_EDGE), Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 20 (BROADCAST_EDGE), Reducer 23 (BROADCAST_EDGE) + Map 1 <- Map 21 (BROADCAST_EDGE), Map 31 (BROADCAST_EDGE), Reducer 11 (BROADCAST_EDGE) + Map 15 <- Reducer 32 (BROADCAST_EDGE), Reducer 33 (BROADCAST_EDGE) + Map 27 <- Map 21 (BROADCAST_EDGE), Map 31 (BROADCAST_EDGE), Reducer 14 (BROADCAST_EDGE), Reducer 7 (BROADCAST_EDGE) + Map 35 <- Reducer 11 (BROADCAST_EDGE), Reducer 14 (BROADCAST_EDGE), Reducer 32 (BROADCAST_EDGE), Reducer 33 (BROADCAST_EDGE) + Map 8 <- Reducer 32 (BROADCAST_EDGE), Reducer 33 (BROADCAST_EDGE) + Reducer 10 <- Reducer 9 (SIMPLE_EDGE) + Reducer 11 <- Reducer 10 (CUSTOM_SIMPLE_EDGE) + Reducer 12 <- Map 15 (CUSTOM_SIMPLE_EDGE), Map 8 (CUSTOM_SIMPLE_EDGE) + Reducer 13 <- Reducer 12 (SIMPLE_EDGE) + Reducer 14 <- Reducer 13 (CUSTOM_SIMPLE_EDGE) + Reducer 19 <- Map 18 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 16 (BROADCAST_EDGE), Map 34 (CUSTOM_SIMPLE_EDGE), Reducer 10 (BROADCAST_EDGE) + Reducer 20 <- Map 18 (SIMPLE_EDGE) + Reducer 22 <- Map 21 (SIMPLE_EDGE) + Reducer 23 <- Map 21 (SIMPLE_EDGE) + Reducer 25 <- Map 24 (SIMPLE_EDGE) + Reducer 26 <- Map 24 (SIMPLE_EDGE) + Reducer 28 <- Map 16 (BROADCAST_EDGE), Map 27 (CUSTOM_SIMPLE_EDGE), Map 34 (CUSTOM_SIMPLE_EDGE), Reducer 13 (BROADCAST_EDGE) + Reducer 29 <- Map 16 (BROADCAST_EDGE), Map 17 (BROADCAST_EDGE), Map 18 (BROADCAST_EDGE), Map 21 (BROADCAST_EDGE), Map 24 (BROADCAST_EDGE), Map 35 (CUSTOM_SIMPLE_EDGE), Reducer 20 (BROADCAST_EDGE), Reducer 23 (BROADCAST_EDGE), Reducer 26 (BROADCAST_EDGE), Reducer 28 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Map 16 (BROADCAST_EDGE), Map 17 (BROADCAST_EDGE), Map 18 (BROADCAST_EDGE), Map 21 (BROADCAST_EDGE), Map 24 (BROADCAST_EDGE), Map 35 (CUSTOM_SIMPLE_EDGE), Reducer 19 (BROADCAST_EDGE), Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 22 (BROADCAST_EDGE), Reducer 25 (BROADCAST_EDGE) Reducer 30 <- Reducer 29 (SIMPLE_EDGE) Reducer 32 <- Map 31 (CUSTOM_SIMPLE_EDGE) Reducer 33 <- Map 31 (CUSTOM_SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 13 (CUSTOM_SIMPLE_EDGE), Map 6 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (CUSTOM_SIMPLE_EDGE) + Reducer 5 <- Reducer 30 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) + Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Reducer 7 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) + Reducer 9 <- Map 15 (CUSTOM_SIMPLE_EDGE), Map 8 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -57,30 +57,30 @@ STAGE PLANS: keys: 0 _col0 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 input vertices: 1 Map 31 - Statistics: Num rows: 1300511220 Data size: 41616359416 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1300511220 Data size: 180771059956 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col10 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col12 input vertices: - 1 Map 19 - Statistics: Num rows: 261380636 Data size: 6273135640 Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 21 + Statistics: Num rows: 261380636 Data size: 34240863692 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: bigint) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: bigint) - Statistics: Num rows: 261380636 Data size: 6273135640 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: decimal(7,2)), _col8 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col11 (type: bigint) + Statistics: Num rows: 261380636 Data size: 34240863692 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: decimal(7,2)), _col8 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col11 (type: bigint), _col12 (type: char(50)) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) - Map 13 + Map 15 Map Operator Tree: TableScan alias: catalog_returns @@ -116,7 +116,7 @@ STAGE PLANS: value expressions: _col2 (type: decimal(9,2)) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) - Map 14 + Map 16 Map Operator Tree: TableScan alias: ad1 @@ -155,7 +155,7 @@ STAGE PLANS: value expressions: _col1 (type: char(10)), _col2 (type: varchar(60)), _col3 (type: varchar(60)), _col4 (type: char(10)) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) - Map 15 + Map 17 Map Operator Tree: TableScan alias: store @@ -184,7 +184,7 @@ STAGE PLANS: value expressions: _col1 (type: varchar(50)), _col2 (type: char(10)) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) - Map 16 + Map 18 Map Operator Tree: TableScan alias: hd1 @@ -223,7 +223,7 @@ STAGE PLANS: Statistics: Num rows: 7200 Data size: 57600 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: may be used (ACID table) - Map 19 + Map 21 Map Operator Tree: TableScan alias: d2 @@ -261,7 +261,7 @@ STAGE PLANS: Statistics: Num rows: 73049 Data size: 876588 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) Filter Operator - predicate: (d_year = 2000) (type: boolean) + predicate: (d_year = 2001) (type: boolean) Statistics: Num rows: 367 Data size: 4404 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: d_date_sk (type: bigint) @@ -288,9 +288,9 @@ STAGE PLANS: Target Input: store_sales Partition key expr: ss_sold_date_sk Statistics: Num rows: 367 Data size: 2936 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 25 + Target Vertex: Map 27 Filter Operator - predicate: (d_year = 2001) (type: boolean) + predicate: (d_year = 2000) (type: boolean) Statistics: Num rows: 367 Data size: 4404 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: d_date_sk (type: bigint) @@ -320,7 +320,7 @@ STAGE PLANS: Target Vertex: Map 1 Execution mode: vectorized, llap LLAP IO: may be used (ACID table) - Map 22 + Map 24 Map Operator Tree: TableScan alias: cd1 @@ -359,7 +359,7 @@ STAGE PLANS: value expressions: _col1 (type: char(1)) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) - Map 25 + Map 27 Map Operator Tree: TableScan alias: store_sales @@ -379,27 +379,27 @@ STAGE PLANS: keys: 0 _col0 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 input vertices: 1 Map 31 - Statistics: Num rows: 1300511220 Data size: 180771059956 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1300511220 Data size: 41616359416 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col10 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col12 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 input vertices: - 1 Map 19 - Statistics: Num rows: 261380636 Data size: 34240863692 Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 21 + Statistics: Num rows: 261380636 Data size: 6273135640 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: bigint) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: bigint) - Statistics: Num rows: 261380636 Data size: 34240863692 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: decimal(7,2)), _col8 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col11 (type: bigint), _col12 (type: char(50)) + Statistics: Num rows: 261380636 Data size: 6273135640 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: decimal(7,2)), _col8 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col11 (type: bigint) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) Map 31 @@ -407,21 +407,20 @@ STAGE PLANS: TableScan alias: item filterExpr: (i_current_price BETWEEN 36 AND 45 and (i_color) IN ('burnished ', 'chocolate ', 'dim ', 'maroon ', 'navajo ', 'steel ')) (type: boolean) - Statistics: Num rows: 462000 Data size: 145861408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 462000 Data size: 96427408 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (i_current_price BETWEEN 36 AND 45 and (i_color) IN ('burnished ', 'chocolate ', 'dim ', 'maroon ', 'navajo ', 'steel ')) (type: boolean) - Statistics: Num rows: 8402 Data size: 2652792 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8402 Data size: 1753778 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: i_item_sk (type: bigint), i_product_name (type: char(50)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 8402 Data size: 966230 Basic stats: COMPLETE Column stats: COMPLETE + expressions: i_item_sk (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 8402 Data size: 67216 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: bigint) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 8402 Data size: 966230 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: char(50)) + Statistics: Num rows: 8402 Data size: 67216 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: bigint) outputColumnNames: _col0 @@ -438,15 +437,16 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: binary) Select Operator - expressions: i_item_sk (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 8402 Data size: 67216 Basic stats: COMPLETE Column stats: COMPLETE + expressions: i_item_sk (type: bigint), i_product_name (type: char(50)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8402 Data size: 966230 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: bigint) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 8402 Data size: 67216 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8402 Data size: 966230 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: char(50)) Select Operator expressions: _col0 (type: bigint) outputColumnNames: _col0 @@ -527,7 +527,7 @@ STAGE PLANS: Statistics: Num rows: 8634166995 Data size: 138146671920 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: may be used (ACID table) - Map 6 + Map 8 Map Operator Tree: TableScan alias: catalog_sales @@ -564,6 +564,61 @@ STAGE PLANS: Execution mode: vectorized, llap LLAP IO: may be used (ACID table) Reducer 10 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1) + keys: KEY._col0 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 447635 Data size: 103851320 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (_col1 > (2 * _col2)) (type: boolean) + Statistics: Num rows: 149211 Data size: 34616952 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 149211 Data size: 1193688 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 149211 Data size: 1193688 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 149211 Data size: 1193688 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000000) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: binary) + Reducer 11 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, 1, expectedEntries=1000000) + mode: final + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: binary) + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: binary) + Reducer 12 Execution mode: vectorized, llap Reduce Operator Tree: Map Join Operator @@ -574,7 +629,7 @@ STAGE PLANS: 1 KEY.reducesinkkey0 (type: bigint), KEY.reducesinkkey1 (type: bigint) outputColumnNames: _col0, _col2, _col5 input vertices: - 1 Map 13 + 1 Map 15 Statistics: Num rows: 41876960211 Data size: 9691486353656 Basic stats: COMPLETE Column stats: COMPLETE DynamicPartitionHashJoin: true Group By Operator @@ -591,7 +646,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 16946565830 Data size: 3931603272560 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: decimal(17,2)), _col2 (type: decimal(19,2)) - Reducer 11 + Reducer 13 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -628,7 +683,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: binary) - Reducer 12 + Reducer 14 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -646,19 +701,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: binary) - Reducer 17 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: bigint) - outputColumnNames: _col0 - Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 7200 Data size: 57600 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Reducer 19 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -679,10 +722,10 @@ STAGE PLANS: keys: 0 KEY.reducesinkkey0 (type: bigint) 1 KEY.reducesinkkey0 (type: bigint) - outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col15, _col16, _col17, _col18, _col19 + outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col12, _col15, _col16, _col17, _col18, _col19 input vertices: 1 Map 34 - Statistics: Num rows: 226670367 Data size: 14429217296 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 226670367 Data size: 38682946565 Basic stats: COMPLETE Column stats: COMPLETE DynamicPartitionHashJoin: true Map Join Operator condition map: @@ -690,28 +733,40 @@ STAGE PLANS: keys: 0 _col0 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col15, _col16, _col17, _col18, _col19 + outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col12, _col15, _col16, _col17, _col18, _col19 input vertices: - 1 Reducer 8 - Statistics: Num rows: 226670367 Data size: 14429217296 Basic stats: COMPLETE Column stats: COMPLETE + 1 Reducer 10 + Statistics: Num rows: 226670367 Data size: 38682946565 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col4 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col0, _col2, _col3, _col5, _col6, _col7, _col8, _col9, _col11, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25 + outputColumnNames: _col0, _col2, _col3, _col5, _col6, _col7, _col8, _col9, _col11, _col12, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25 input vertices: - 1 Map 14 - Statistics: Num rows: 226670367 Data size: 96257219775 Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 16 + Statistics: Num rows: 226670367 Data size: 120510949044 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: bigint), _col6 (type: bigint) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: bigint), _col6 (type: bigint) - Statistics: Num rows: 226670367 Data size: 96257219775 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint), _col3 (type: bigint), _col5 (type: bigint), _col7 (type: decimal(7,2)), _col8 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col11 (type: bigint), _col15 (type: bigint), _col16 (type: bigint), _col17 (type: bigint), _col18 (type: bigint), _col19 (type: bigint), _col22 (type: char(10)), _col23 (type: varchar(60)), _col24 (type: varchar(60)), _col25 (type: char(10)) + Statistics: Num rows: 226670367 Data size: 120510949044 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint), _col3 (type: bigint), _col5 (type: bigint), _col7 (type: decimal(7,2)), _col8 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col11 (type: bigint), _col12 (type: char(50)), _col15 (type: bigint), _col16 (type: bigint), _col17 (type: bigint), _col18 (type: bigint), _col19 (type: bigint), _col22 (type: char(10)), _col23 (type: varchar(60)), _col24 (type: varchar(60)), _col25 (type: char(10)) Reducer 20 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: bigint) + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: bigint) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 7200 Data size: 57600 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 22 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -724,7 +779,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 73049 Data size: 876588 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 21 + Reducer 23 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -737,7 +792,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 73049 Data size: 876588 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 23 + Reducer 25 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -750,7 +805,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1920800 Data size: 178634400 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: char(1)) - Reducer 24 + Reducer 26 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -763,7 +818,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1920800 Data size: 178634400 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: char(1)) - Reducer 26 + Reducer 28 Execution mode: vectorized, llap Reduce Operator Tree: Map Join Operator @@ -772,10 +827,10 @@ STAGE PLANS: keys: 0 KEY.reducesinkkey0 (type: bigint) 1 KEY.reducesinkkey0 (type: bigint) - outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col12, _col15, _col16, _col17, _col18, _col19 + outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col15, _col16, _col17, _col18, _col19 input vertices: 1 Map 34 - Statistics: Num rows: 226670367 Data size: 38682946565 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 226670367 Data size: 14429217296 Basic stats: COMPLETE Column stats: COMPLETE DynamicPartitionHashJoin: true Map Join Operator condition map: @@ -783,28 +838,28 @@ STAGE PLANS: keys: 0 _col0 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col12, _col15, _col16, _col17, _col18, _col19 + outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col15, _col16, _col17, _col18, _col19 input vertices: - 1 Reducer 11 - Statistics: Num rows: 226670367 Data size: 38682946565 Basic stats: COMPLETE Column stats: COMPLETE + 1 Reducer 13 + Statistics: Num rows: 226670367 Data size: 14429217296 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col4 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col0, _col2, _col3, _col5, _col6, _col7, _col8, _col9, _col11, _col12, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25 + outputColumnNames: _col0, _col2, _col3, _col5, _col6, _col7, _col8, _col9, _col11, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25 input vertices: - 1 Map 14 - Statistics: Num rows: 226670367 Data size: 120510949044 Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 16 + Statistics: Num rows: 226670367 Data size: 96257219775 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: bigint), _col6 (type: bigint) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: bigint), _col6 (type: bigint) - Statistics: Num rows: 226670367 Data size: 120510949044 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint), _col3 (type: bigint), _col5 (type: bigint), _col7 (type: decimal(7,2)), _col8 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col11 (type: bigint), _col12 (type: char(50)), _col15 (type: bigint), _col16 (type: bigint), _col17 (type: bigint), _col18 (type: bigint), _col19 (type: bigint), _col22 (type: char(10)), _col23 (type: varchar(60)), _col24 (type: varchar(60)), _col25 (type: char(10)) - Reducer 27 + Statistics: Num rows: 226670367 Data size: 96257219775 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint), _col3 (type: bigint), _col5 (type: bigint), _col7 (type: decimal(7,2)), _col8 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col11 (type: bigint), _col15 (type: bigint), _col16 (type: bigint), _col17 (type: bigint), _col18 (type: bigint), _col19 (type: bigint), _col22 (type: char(10)), _col23 (type: varchar(60)), _col24 (type: varchar(60)), _col25 (type: char(10)) + Reducer 29 Execution mode: vectorized, llap Reduce Operator Tree: Map Join Operator @@ -813,10 +868,10 @@ STAGE PLANS: keys: 0 KEY.reducesinkkey0 (type: bigint), KEY.reducesinkkey1 (type: bigint) 1 KEY.reducesinkkey0 (type: bigint), KEY.reducesinkkey1 (type: bigint) - outputColumnNames: _col2, _col3, _col5, _col7, _col8, _col9, _col11, _col12, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25 + outputColumnNames: _col2, _col3, _col5, _col7, _col8, _col9, _col11, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25 input vertices: 1 Map 35 - Statistics: Num rows: 382653083 Data size: 253525082388 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 382653083 Data size: 212581202507 Basic stats: COMPLETE Column stats: COMPLETE DynamicPartitionHashJoin: true Map Join Operator condition map: @@ -824,151 +879,97 @@ STAGE PLANS: keys: 0 _col5 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col2, _col3, _col7, _col8, _col9, _col11, _col12, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col29, _col30 + outputColumnNames: _col2, _col3, _col7, _col8, _col9, _col11, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col29, _col30 input vertices: - 1 Map 15 - Statistics: Num rows: 382653083 Data size: 320006816343 Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 17 + Statistics: Num rows: 382653083 Data size: 279062936462 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col3 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col2, _col7, _col8, _col9, _col11, _col12, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col29, _col30 + outputColumnNames: _col2, _col7, _col8, _col9, _col11, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col29, _col30 input vertices: - 1 Reducer 18 - Statistics: Num rows: 382653083 Data size: 318758954607 Basic stats: COMPLETE Column stats: COMPLETE + 1 Reducer 20 + Statistics: Num rows: 382653083 Data size: 277815074726 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col16 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col2, _col7, _col8, _col9, _col11, _col12, _col15, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col29, _col30 + outputColumnNames: _col2, _col7, _col8, _col9, _col11, _col15, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col29, _col30 input vertices: - 1 Map 16 - Statistics: Num rows: 382653083 Data size: 315717148415 Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 18 + Statistics: Num rows: 382653083 Data size: 274773268534 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col19 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col2, _col7, _col8, _col9, _col11, _col12, _col15, _col17, _col18, _col22, _col23, _col24, _col25, _col29, _col30, _col34 + outputColumnNames: _col2, _col7, _col8, _col9, _col11, _col15, _col17, _col18, _col22, _col23, _col24, _col25, _col29, _col30, _col34 input vertices: - 1 Reducer 21 - Statistics: Num rows: 382653083 Data size: 314205958075 Basic stats: COMPLETE Column stats: COMPLETE + 1 Reducer 23 + Statistics: Num rows: 382653083 Data size: 273262078194 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col18 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col2, _col7, _col8, _col9, _col11, _col12, _col15, _col17, _col22, _col23, _col24, _col25, _col29, _col30, _col34, _col36 + outputColumnNames: _col2, _col7, _col8, _col9, _col11, _col15, _col17, _col22, _col23, _col24, _col25, _col29, _col30, _col34, _col36 input vertices: - 1 Map 19 - Statistics: Num rows: 382653083 Data size: 312694776079 Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 21 + Statistics: Num rows: 382653083 Data size: 271750896198 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col2 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col7, _col8, _col9, _col11, _col12, _col15, _col17, _col22, _col23, _col24, _col25, _col29, _col30, _col34, _col36, _col38 + outputColumnNames: _col7, _col8, _col9, _col11, _col15, _col17, _col22, _col23, _col24, _col25, _col29, _col30, _col34, _col36, _col38 input vertices: - 1 Reducer 24 - Statistics: Num rows: 382653083 Data size: 343972426398 Basic stats: COMPLETE Column stats: COMPLETE + 1 Reducer 26 + Statistics: Num rows: 382653083 Data size: 303028546517 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col15 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col7, _col8, _col9, _col11, _col12, _col17, _col22, _col23, _col24, _col25, _col29, _col30, _col34, _col36, _col38, _col40 + outputColumnNames: _col7, _col8, _col9, _col11, _col17, _col22, _col23, _col24, _col25, _col29, _col30, _col34, _col36, _col38, _col40 input vertices: - 1 Map 22 - Statistics: Num rows: 382653083 Data size: 373456129549 Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 24 + Statistics: Num rows: 382653083 Data size: 332512249668 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (_col38 <> _col40) (type: boolean) - Statistics: Num rows: 382653083 Data size: 373456129549 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 382653083 Data size: 332512249668 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col17 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col7, _col8, _col9, _col11, _col12, _col22, _col23, _col24, _col25, _col29, _col30, _col34, _col36, _col42, _col43, _col44, _col45 + outputColumnNames: _col7, _col8, _col9, _col11, _col22, _col23, _col24, _col25, _col29, _col30, _col34, _col36, _col42, _col43, _col44, _col45 input vertices: - 1 Map 14 - Statistics: Num rows: 382653083 Data size: 443481643738 Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 16 + Statistics: Num rows: 382653083 Data size: 402537763857 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), sum(_col7), sum(_col8), sum(_col9) - keys: _col29 (type: varchar(50)), _col11 (type: bigint), _col30 (type: char(10)), _col22 (type: char(10)), _col23 (type: varchar(60)), _col24 (type: varchar(60)), _col25 (type: char(10)), _col12 (type: char(50)), _col34 (type: int), _col36 (type: int), _col42 (type: char(10)), _col43 (type: varchar(60)), _col44 (type: varchar(60)), _col45 (type: char(10)) - minReductionHashAggr: 0.823521 + keys: _col29 (type: varchar(50)), _col11 (type: bigint), _col30 (type: char(10)), _col22 (type: char(10)), _col23 (type: varchar(60)), _col24 (type: varchar(60)), _col25 (type: char(10)), _col34 (type: int), _col36 (type: int), _col42 (type: char(10)), _col43 (type: varchar(60)), _col44 (type: varchar(60)), _col45 (type: char(10)) + minReductionHashAggr: 0.8260248 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 382653083 Data size: 522704111378 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 382653083 Data size: 481760231497 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: varchar(50)), _col1 (type: bigint), _col2 (type: char(10)), _col3 (type: char(10)), _col4 (type: varchar(60)), _col5 (type: varchar(60)), _col6 (type: char(10)), _col7 (type: char(50)), _col8 (type: int), _col9 (type: int), _col10 (type: char(10)), _col11 (type: varchar(60)), _col12 (type: varchar(60)), _col13 (type: char(10)) - null sort order: zzzzzzzzzzzzzz - sort order: ++++++++++++++ - Map-reduce partition columns: _col0 (type: varchar(50)), _col1 (type: bigint), _col2 (type: char(10)), _col3 (type: char(10)), _col4 (type: varchar(60)), _col5 (type: varchar(60)), _col6 (type: char(10)), _col7 (type: char(50)), _col8 (type: int), _col9 (type: int), _col10 (type: char(10)), _col11 (type: varchar(60)), _col12 (type: varchar(60)), _col13 (type: char(10)) - Statistics: Num rows: 382653083 Data size: 522704111378 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col14 (type: bigint), _col15 (type: decimal(17,2)), _col16 (type: decimal(17,2)), _col17 (type: decimal(17,2)) - Reducer 28 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3) - keys: KEY._col0 (type: varchar(50)), KEY._col1 (type: bigint), KEY._col2 (type: char(10)), KEY._col3 (type: char(10)), KEY._col4 (type: varchar(60)), KEY._col5 (type: varchar(60)), KEY._col6 (type: char(10)), KEY._col7 (type: char(50)), KEY._col8 (type: int), KEY._col9 (type: int), KEY._col10 (type: char(10)), KEY._col11 (type: varchar(60)), KEY._col12 (type: varchar(60)), KEY._col13 (type: char(10)) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 382653083 Data size: 522704111378 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col7 (type: char(50)), _col1 (type: bigint), _col0 (type: varchar(50)), _col2 (type: char(10)), _col3 (type: char(10)), _col4 (type: varchar(60)), _col5 (type: varchar(60)), _col6 (type: char(10)), _col10 (type: char(10)), _col11 (type: varchar(60)), _col12 (type: varchar(60)), _col13 (type: char(10)), _col14 (type: bigint), _col15 (type: decimal(17,2)), _col16 (type: decimal(17,2)), _col17 (type: decimal(17,2)) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col14, _col15, _col16, _col17 - Statistics: Num rows: 382653083 Data size: 519642886714 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col14 is not null (type: boolean) - Statistics: Num rows: 382653083 Data size: 519642886714 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: char(50)), _col1 (type: bigint), _col2 (type: varchar(50)), _col3 (type: char(10)), _col4 (type: char(10)), _col5 (type: varchar(60)), _col6 (type: varchar(60)), _col7 (type: char(10)), _col8 (type: char(10)), _col9 (type: varchar(60)), _col10 (type: varchar(60)), _col11 (type: char(10)), _col14 (type: bigint), _col15 (type: decimal(17,2)), _col16 (type: decimal(17,2)), _col17 (type: decimal(17,2)) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15 - Statistics: Num rows: 382653083 Data size: 519642886714 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: varchar(50)), _col1 (type: bigint), _col3 (type: char(10)) - null sort order: zzz - sort order: +++ - Map-reduce partition columns: _col2 (type: varchar(50)), _col1 (type: bigint), _col3 (type: char(10)) - Statistics: Num rows: 382653083 Data size: 519642886714 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: char(50)), _col4 (type: char(10)), _col5 (type: varchar(60)), _col6 (type: varchar(60)), _col7 (type: char(10)), _col8 (type: char(10)), _col9 (type: varchar(60)), _col10 (type: varchar(60)), _col11 (type: char(10)), _col12 (type: bigint), _col13 (type: decimal(17,2)), _col14 (type: decimal(17,2)), _col15 (type: decimal(17,2)) - Reducer 29 - Execution mode: vectorized, llap - Reduce Operator Tree: - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 KEY.reducesinkkey0 (type: varchar(50)), KEY.reducesinkkey1 (type: bigint), KEY.reducesinkkey2 (type: char(10)) - 1 KEY.reducesinkkey0 (type: varchar(50)), KEY.reducesinkkey1 (type: bigint), KEY.reducesinkkey2 (type: char(10)) - outputColumnNames: _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22 - input vertices: - 0 Reducer 4 - Statistics: Num rows: 99947700975703 Data size: 169311405452840883 Basic stats: COMPLETE Column stats: COMPLETE - DynamicPartitionHashJoin: true - Filter Operator - predicate: (_col3 <= _col19) (type: boolean) - Statistics: Num rows: 33315900325234 Data size: 56437135150946396 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col7 (type: char(50)), _col9 (type: varchar(50)), _col10 (type: char(10)), _col11 (type: char(10)), _col12 (type: varchar(60)), _col13 (type: varchar(60)), _col14 (type: char(10)), _col15 (type: char(10)), _col16 (type: varchar(60)), _col17 (type: varchar(60)), _col18 (type: char(10)), _col19 (type: bigint), _col20 (type: decimal(17,2)), _col21 (type: decimal(17,2)), _col22 (type: decimal(17,2)), _col4 (type: decimal(17,2)), _col5 (type: decimal(17,2)), _col6 (type: decimal(17,2)), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 33315900325234 Data size: 56437135150946396 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: char(50)), _col1 (type: varchar(50)), _col18 (type: bigint) - null sort order: zzz - sort order: +++ - Statistics: Num rows: 33315900325234 Data size: 56437135150946396 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: char(10)), _col3 (type: char(10)), _col4 (type: varchar(60)), _col5 (type: varchar(60)), _col6 (type: char(10)), _col7 (type: char(10)), _col8 (type: varchar(60)), _col9 (type: varchar(60)), _col10 (type: char(10)), _col11 (type: bigint), _col12 (type: decimal(17,2)), _col13 (type: decimal(17,2)), _col14 (type: decimal(17,2)), _col15 (type: decimal(17,2)), _col16 (type: decimal(17,2)), _col17 (type: decimal(17,2)) + key expressions: _col0 (type: varchar(50)), _col1 (type: bigint), _col2 (type: char(10)), _col3 (type: char(10)), _col4 (type: varchar(60)), _col5 (type: varchar(60)), _col6 (type: char(10)), _col7 (type: int), _col8 (type: int), _col9 (type: char(10)), _col10 (type: varchar(60)), _col11 (type: varchar(60)), _col12 (type: char(10)) + null sort order: zzzzzzzzzzzzz + sort order: +++++++++++++ + Map-reduce partition columns: _col0 (type: varchar(50)), _col1 (type: bigint), _col2 (type: char(10)), _col3 (type: char(10)), _col4 (type: varchar(60)), _col5 (type: varchar(60)), _col6 (type: char(10)), _col7 (type: int), _col8 (type: int), _col9 (type: char(10)), _col10 (type: varchar(60)), _col11 (type: varchar(60)), _col12 (type: char(10)) + Statistics: Num rows: 382653083 Data size: 481760231497 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col13 (type: bigint), _col14 (type: decimal(17,2)), _col15 (type: decimal(17,2)), _col16 (type: decimal(17,2)) Reducer 3 Execution mode: vectorized, llap Reduce Operator Tree: @@ -978,10 +979,10 @@ STAGE PLANS: keys: 0 KEY.reducesinkkey0 (type: bigint), KEY.reducesinkkey1 (type: bigint) 1 KEY.reducesinkkey0 (type: bigint), KEY.reducesinkkey1 (type: bigint) - outputColumnNames: _col2, _col3, _col5, _col7, _col8, _col9, _col11, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25 + outputColumnNames: _col2, _col3, _col5, _col7, _col8, _col9, _col11, _col12, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25 input vertices: 1 Map 35 - Statistics: Num rows: 382653083 Data size: 212581202507 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 382653083 Data size: 253525082388 Basic stats: COMPLETE Column stats: COMPLETE DynamicPartitionHashJoin: true Map Join Operator condition map: @@ -989,111 +990,124 @@ STAGE PLANS: keys: 0 _col5 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col2, _col3, _col7, _col8, _col9, _col11, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col29, _col30 + outputColumnNames: _col2, _col3, _col7, _col8, _col9, _col11, _col12, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col29, _col30 input vertices: - 1 Map 15 - Statistics: Num rows: 382653083 Data size: 279062936462 Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 17 + Statistics: Num rows: 382653083 Data size: 320006816343 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col3 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col2, _col7, _col8, _col9, _col11, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col29, _col30 + outputColumnNames: _col2, _col7, _col8, _col9, _col11, _col12, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col29, _col30 input vertices: - 1 Reducer 17 - Statistics: Num rows: 382653083 Data size: 277815074726 Basic stats: COMPLETE Column stats: COMPLETE + 1 Reducer 19 + Statistics: Num rows: 382653083 Data size: 318758954607 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col16 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col2, _col7, _col8, _col9, _col11, _col15, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col29, _col30 + outputColumnNames: _col2, _col7, _col8, _col9, _col11, _col12, _col15, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col29, _col30 input vertices: - 1 Map 16 - Statistics: Num rows: 382653083 Data size: 274773268534 Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 18 + Statistics: Num rows: 382653083 Data size: 315717148415 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col19 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col2, _col7, _col8, _col9, _col11, _col15, _col17, _col18, _col22, _col23, _col24, _col25, _col29, _col30, _col34 + outputColumnNames: _col2, _col7, _col8, _col9, _col11, _col12, _col15, _col17, _col18, _col22, _col23, _col24, _col25, _col29, _col30, _col34 input vertices: - 1 Reducer 20 - Statistics: Num rows: 382653083 Data size: 273262078194 Basic stats: COMPLETE Column stats: COMPLETE + 1 Reducer 22 + Statistics: Num rows: 382653083 Data size: 314205958075 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col18 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col2, _col7, _col8, _col9, _col11, _col15, _col17, _col22, _col23, _col24, _col25, _col29, _col30, _col34, _col36 + outputColumnNames: _col2, _col7, _col8, _col9, _col11, _col12, _col15, _col17, _col22, _col23, _col24, _col25, _col29, _col30, _col34, _col36 input vertices: - 1 Map 19 - Statistics: Num rows: 382653083 Data size: 271750896198 Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 21 + Statistics: Num rows: 382653083 Data size: 312694776079 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col2 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col7, _col8, _col9, _col11, _col15, _col17, _col22, _col23, _col24, _col25, _col29, _col30, _col34, _col36, _col38 + outputColumnNames: _col7, _col8, _col9, _col11, _col12, _col15, _col17, _col22, _col23, _col24, _col25, _col29, _col30, _col34, _col36, _col38 input vertices: - 1 Reducer 23 - Statistics: Num rows: 382653083 Data size: 303028546517 Basic stats: COMPLETE Column stats: COMPLETE + 1 Reducer 25 + Statistics: Num rows: 382653083 Data size: 343972426398 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col15 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col7, _col8, _col9, _col11, _col17, _col22, _col23, _col24, _col25, _col29, _col30, _col34, _col36, _col38, _col40 + outputColumnNames: _col7, _col8, _col9, _col11, _col12, _col17, _col22, _col23, _col24, _col25, _col29, _col30, _col34, _col36, _col38, _col40 input vertices: - 1 Map 22 - Statistics: Num rows: 382653083 Data size: 332512249668 Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 24 + Statistics: Num rows: 382653083 Data size: 373456129549 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (_col38 <> _col40) (type: boolean) - Statistics: Num rows: 382653083 Data size: 332512249668 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 382653083 Data size: 373456129549 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col17 (type: bigint) 1 _col0 (type: bigint) - outputColumnNames: _col7, _col8, _col9, _col11, _col22, _col23, _col24, _col25, _col29, _col30, _col34, _col36, _col42, _col43, _col44, _col45 + outputColumnNames: _col7, _col8, _col9, _col11, _col12, _col22, _col23, _col24, _col25, _col29, _col30, _col34, _col36, _col42, _col43, _col44, _col45 input vertices: - 1 Map 14 - Statistics: Num rows: 382653083 Data size: 402537763857 Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 16 + Statistics: Num rows: 382653083 Data size: 443481643738 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), sum(_col7), sum(_col8), sum(_col9) - keys: _col29 (type: varchar(50)), _col11 (type: bigint), _col30 (type: char(10)), _col22 (type: char(10)), _col23 (type: varchar(60)), _col24 (type: varchar(60)), _col25 (type: char(10)), _col34 (type: int), _col36 (type: int), _col42 (type: char(10)), _col43 (type: varchar(60)), _col44 (type: varchar(60)), _col45 (type: char(10)) - minReductionHashAggr: 0.8260248 + keys: _col29 (type: varchar(50)), _col11 (type: bigint), _col30 (type: char(10)), _col22 (type: char(10)), _col23 (type: varchar(60)), _col24 (type: varchar(60)), _col25 (type: char(10)), _col12 (type: char(50)), _col34 (type: int), _col36 (type: int), _col42 (type: char(10)), _col43 (type: varchar(60)), _col44 (type: varchar(60)), _col45 (type: char(10)) + minReductionHashAggr: 0.823521 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 382653083 Data size: 481760231497 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 382653083 Data size: 522704111378 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: varchar(50)), _col1 (type: bigint), _col2 (type: char(10)), _col3 (type: char(10)), _col4 (type: varchar(60)), _col5 (type: varchar(60)), _col6 (type: char(10)), _col7 (type: int), _col8 (type: int), _col9 (type: char(10)), _col10 (type: varchar(60)), _col11 (type: varchar(60)), _col12 (type: char(10)) - null sort order: zzzzzzzzzzzzz - sort order: +++++++++++++ - Map-reduce partition columns: _col0 (type: varchar(50)), _col1 (type: bigint), _col2 (type: char(10)), _col3 (type: char(10)), _col4 (type: varchar(60)), _col5 (type: varchar(60)), _col6 (type: char(10)), _col7 (type: int), _col8 (type: int), _col9 (type: char(10)), _col10 (type: varchar(60)), _col11 (type: varchar(60)), _col12 (type: char(10)) - Statistics: Num rows: 382653083 Data size: 481760231497 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col13 (type: bigint), _col14 (type: decimal(17,2)), _col15 (type: decimal(17,2)), _col16 (type: decimal(17,2)) + key expressions: _col0 (type: varchar(50)), _col1 (type: bigint), _col2 (type: char(10)), _col3 (type: char(10)), _col4 (type: varchar(60)), _col5 (type: varchar(60)), _col6 (type: char(10)), _col7 (type: char(50)), _col8 (type: int), _col9 (type: int), _col10 (type: char(10)), _col11 (type: varchar(60)), _col12 (type: varchar(60)), _col13 (type: char(10)) + null sort order: zzzzzzzzzzzzzz + sort order: ++++++++++++++ + Map-reduce partition columns: _col0 (type: varchar(50)), _col1 (type: bigint), _col2 (type: char(10)), _col3 (type: char(10)), _col4 (type: varchar(60)), _col5 (type: varchar(60)), _col6 (type: char(10)), _col7 (type: char(50)), _col8 (type: int), _col9 (type: int), _col10 (type: char(10)), _col11 (type: varchar(60)), _col12 (type: varchar(60)), _col13 (type: char(10)) + Statistics: Num rows: 382653083 Data size: 522704111378 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col14 (type: bigint), _col15 (type: decimal(17,2)), _col16 (type: decimal(17,2)), _col17 (type: decimal(17,2)) Reducer 30 Execution mode: vectorized, llap Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: char(50)), KEY.reducesinkkey1 (type: varchar(50)), VALUE._col0 (type: char(10)), VALUE._col1 (type: char(10)), VALUE._col2 (type: varchar(60)), VALUE._col3 (type: varchar(60)), VALUE._col4 (type: char(10)), VALUE._col5 (type: char(10)), VALUE._col6 (type: varchar(60)), VALUE._col7 (type: varchar(60)), VALUE._col8 (type: char(10)), 2000 (type: int), VALUE._col9 (type: bigint), VALUE._col10 (type: decimal(17,2)), VALUE._col11 (type: decimal(17,2)), VALUE._col12 (type: decimal(17,2)), VALUE._col13 (type: decimal(17,2)), VALUE._col14 (type: decimal(17,2)), VALUE._col15 (type: decimal(17,2)), 2001 (type: int), KEY.reducesinkkey2 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 - Statistics: Num rows: 33315900325234 Data size: 56703662353548268 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 33315900325234 Data size: 56703662353548268 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Group By Operator + aggregations: count(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3) + keys: KEY._col0 (type: varchar(50)), KEY._col1 (type: bigint), KEY._col2 (type: char(10)), KEY._col3 (type: char(10)), KEY._col4 (type: varchar(60)), KEY._col5 (type: varchar(60)), KEY._col6 (type: char(10)), KEY._col7 (type: int), KEY._col8 (type: int), KEY._col9 (type: char(10)), KEY._col10 (type: varchar(60)), KEY._col11 (type: varchar(60)), KEY._col12 (type: char(10)) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 382653083 Data size: 481760231497 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: bigint), _col0 (type: varchar(50)), _col2 (type: char(10)), _col13 (type: bigint), _col14 (type: decimal(17,2)), _col15 (type: decimal(17,2)), _col16 (type: decimal(17,2)) + outputColumnNames: _col0, _col1, _col2, _col13, _col14, _col15, _col16 + Statistics: Num rows: 382653083 Data size: 202423480907 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: _col13 is not null (type: boolean) + Statistics: Num rows: 382653083 Data size: 202423480907 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: varchar(50)), _col2 (type: char(10)), _col13 (type: bigint), _col14 (type: decimal(17,2)), _col15 (type: decimal(17,2)), _col16 (type: decimal(17,2)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 382653083 Data size: 202423480907 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: varchar(50)), _col0 (type: bigint), _col2 (type: char(10)) + null sort order: zzz + sort order: +++ + Map-reduce partition columns: _col1 (type: varchar(50)), _col0 (type: bigint), _col2 (type: char(10)) + Statistics: Num rows: 382653083 Data size: 202423480907 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: decimal(17,2)), _col5 (type: decimal(17,2)), _col6 (type: decimal(17,2)) Reducer 32 Execution mode: vectorized, llap Reduce Operator Tree: @@ -1145,34 +1159,34 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3) - keys: KEY._col0 (type: varchar(50)), KEY._col1 (type: bigint), KEY._col2 (type: char(10)), KEY._col3 (type: char(10)), KEY._col4 (type: varchar(60)), KEY._col5 (type: varchar(60)), KEY._col6 (type: char(10)), KEY._col7 (type: int), KEY._col8 (type: int), KEY._col9 (type: char(10)), KEY._col10 (type: varchar(60)), KEY._col11 (type: varchar(60)), KEY._col12 (type: char(10)) + keys: KEY._col0 (type: varchar(50)), KEY._col1 (type: bigint), KEY._col2 (type: char(10)), KEY._col3 (type: char(10)), KEY._col4 (type: varchar(60)), KEY._col5 (type: varchar(60)), KEY._col6 (type: char(10)), KEY._col7 (type: char(50)), KEY._col8 (type: int), KEY._col9 (type: int), KEY._col10 (type: char(10)), KEY._col11 (type: varchar(60)), KEY._col12 (type: varchar(60)), KEY._col13 (type: char(10)) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 382653083 Data size: 481760231497 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 382653083 Data size: 522704111378 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: bigint), _col0 (type: varchar(50)), _col2 (type: char(10)), _col13 (type: bigint), _col14 (type: decimal(17,2)), _col15 (type: decimal(17,2)), _col16 (type: decimal(17,2)) - outputColumnNames: _col0, _col1, _col2, _col13, _col14, _col15, _col16 - Statistics: Num rows: 382653083 Data size: 202423480907 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col7 (type: char(50)), _col1 (type: bigint), _col0 (type: varchar(50)), _col2 (type: char(10)), _col3 (type: char(10)), _col4 (type: varchar(60)), _col5 (type: varchar(60)), _col6 (type: char(10)), _col10 (type: char(10)), _col11 (type: varchar(60)), _col12 (type: varchar(60)), _col13 (type: char(10)), _col14 (type: bigint), _col15 (type: decimal(17,2)), _col16 (type: decimal(17,2)), _col17 (type: decimal(17,2)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col14, _col15, _col16, _col17 + Statistics: Num rows: 382653083 Data size: 519642886714 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: _col13 is not null (type: boolean) - Statistics: Num rows: 382653083 Data size: 202423480907 Basic stats: COMPLETE Column stats: COMPLETE + predicate: _col14 is not null (type: boolean) + Statistics: Num rows: 382653083 Data size: 519642886714 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: bigint), _col1 (type: varchar(50)), _col2 (type: char(10)), _col13 (type: bigint), _col14 (type: decimal(17,2)), _col15 (type: decimal(17,2)), _col16 (type: decimal(17,2)) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 382653083 Data size: 202423480907 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: char(50)), _col1 (type: bigint), _col2 (type: varchar(50)), _col3 (type: char(10)), _col4 (type: char(10)), _col5 (type: varchar(60)), _col6 (type: varchar(60)), _col7 (type: char(10)), _col8 (type: char(10)), _col9 (type: varchar(60)), _col10 (type: varchar(60)), _col11 (type: char(10)), _col14 (type: bigint), _col15 (type: decimal(17,2)), _col16 (type: decimal(17,2)), _col17 (type: decimal(17,2)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15 + Statistics: Num rows: 382653083 Data size: 519642886714 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col1 (type: varchar(50)), _col0 (type: bigint), _col2 (type: char(10)) + key expressions: _col2 (type: varchar(50)), _col1 (type: bigint), _col3 (type: char(10)) null sort order: zzz sort order: +++ - Map-reduce partition columns: _col1 (type: varchar(50)), _col0 (type: bigint), _col2 (type: char(10)) - Statistics: Num rows: 382653083 Data size: 202423480907 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: decimal(17,2)), _col5 (type: decimal(17,2)), _col6 (type: decimal(17,2)) + Map-reduce partition columns: _col2 (type: varchar(50)), _col1 (type: bigint), _col3 (type: char(10)) + Statistics: Num rows: 382653083 Data size: 519642886714 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: char(50)), _col4 (type: char(10)), _col5 (type: varchar(60)), _col6 (type: varchar(60)), _col7 (type: char(10)), _col8 (type: char(10)), _col9 (type: varchar(60)), _col10 (type: varchar(60)), _col11 (type: char(10)), _col12 (type: bigint), _col13 (type: decimal(17,2)), _col14 (type: decimal(17,2)), _col15 (type: decimal(17,2)) Select Operator - expressions: _col0 (type: bigint) - outputColumnNames: _col0 + expressions: _col1 (type: bigint) + outputColumnNames: _col1 Statistics: Num rows: 382653083 Data size: 3061224664 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000000) + aggregations: min(_col1), max(_col1), bloom_filter(_col1, expectedEntries=1000000) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2 @@ -1183,6 +1197,47 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: binary) Reducer 5 + Execution mode: vectorized, llap + Reduce Operator Tree: + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 KEY.reducesinkkey0 (type: varchar(50)), KEY.reducesinkkey1 (type: bigint), KEY.reducesinkkey2 (type: char(10)) + 1 KEY.reducesinkkey0 (type: varchar(50)), KEY.reducesinkkey1 (type: bigint), KEY.reducesinkkey2 (type: char(10)) + outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col19, _col20, _col21, _col22 + input vertices: + 1 Reducer 30 + Statistics: Num rows: 99947700975703 Data size: 169311405452840883 Basic stats: COMPLETE Column stats: COMPLETE + DynamicPartitionHashJoin: true + Filter Operator + predicate: (_col19 <= _col12) (type: boolean) + Statistics: Num rows: 33315900325234 Data size: 56437135150946396 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: char(50)), _col2 (type: varchar(50)), _col3 (type: char(10)), _col4 (type: char(10)), _col5 (type: varchar(60)), _col6 (type: varchar(60)), _col7 (type: char(10)), _col8 (type: char(10)), _col9 (type: varchar(60)), _col10 (type: varchar(60)), _col11 (type: char(10)), _col12 (type: bigint), _col13 (type: decimal(17,2)), _col14 (type: decimal(17,2)), _col15 (type: decimal(17,2)), _col20 (type: decimal(17,2)), _col21 (type: decimal(17,2)), _col22 (type: decimal(17,2)), _col19 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 + Statistics: Num rows: 33315900325234 Data size: 56437135150946396 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: char(50)), _col1 (type: varchar(50)), _col18 (type: bigint) + null sort order: zzz + sort order: +++ + Statistics: Num rows: 33315900325234 Data size: 56437135150946396 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: char(10)), _col3 (type: char(10)), _col4 (type: varchar(60)), _col5 (type: varchar(60)), _col6 (type: char(10)), _col7 (type: char(10)), _col8 (type: varchar(60)), _col9 (type: varchar(60)), _col10 (type: char(10)), _col11 (type: bigint), _col12 (type: decimal(17,2)), _col13 (type: decimal(17,2)), _col14 (type: decimal(17,2)), _col15 (type: decimal(17,2)), _col16 (type: decimal(17,2)), _col17 (type: decimal(17,2)) + Reducer 6 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: char(50)), KEY.reducesinkkey1 (type: varchar(50)), VALUE._col0 (type: char(10)), VALUE._col1 (type: char(10)), VALUE._col2 (type: varchar(60)), VALUE._col3 (type: varchar(60)), VALUE._col4 (type: char(10)), VALUE._col5 (type: char(10)), VALUE._col6 (type: varchar(60)), VALUE._col7 (type: varchar(60)), VALUE._col8 (type: char(10)), 2000 (type: int), VALUE._col9 (type: bigint), VALUE._col10 (type: decimal(17,2)), VALUE._col11 (type: decimal(17,2)), VALUE._col12 (type: decimal(17,2)), VALUE._col13 (type: decimal(17,2)), VALUE._col14 (type: decimal(17,2)), VALUE._col15 (type: decimal(17,2)), 2001 (type: int), KEY.reducesinkkey2 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 + Statistics: Num rows: 33315900325234 Data size: 56703662353548268 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 33315900325234 Data size: 56703662353548268 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 7 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1195,7 +1250,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: binary) - Reducer 7 + Reducer 9 Execution mode: vectorized, llap Reduce Operator Tree: Map Join Operator @@ -1206,7 +1261,7 @@ STAGE PLANS: 1 KEY.reducesinkkey0 (type: bigint), KEY.reducesinkkey1 (type: bigint) outputColumnNames: _col0, _col2, _col5 input vertices: - 1 Map 13 + 1 Map 15 Statistics: Num rows: 41876960211 Data size: 9691486353656 Basic stats: COMPLETE Column stats: COMPLETE DynamicPartitionHashJoin: true Group By Operator @@ -1223,61 +1278,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 16946565830 Data size: 3931603272560 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: decimal(17,2)), _col2 (type: decimal(19,2)) - Reducer 8 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1) - keys: KEY._col0 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 447635 Data size: 103851320 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (_col1 > (2 * _col2)) (type: boolean) - Statistics: Num rows: 149211 Data size: 34616952 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 149211 Data size: 1193688 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 149211 Data size: 1193688 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 149211 Data size: 1193688 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000000) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: binary) - Reducer 9 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, 1, expectedEntries=1000000) - mode: final - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: binary) - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: binary) Stage: Stage-0 Fetch Operator From 3e32056b957c700d4f497cb0c2ce0324757c7fd5 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Fri, 25 Sep 2026 12:03:56 +0100 Subject: [PATCH 24/24] Remove / simplify some IS_NOT_NULL predicates in cbo_query72.q and cbo_query88.q (probably due to CALCITE-7722) --- .../jdbc/postgres/cbo_query72.q.out | 2 +- .../jdbc/postgres/cbo_query88.q.out | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ql/src/test/results/clientpositive/jdbc/postgres/cbo_query72.q.out b/ql/src/test/results/clientpositive/jdbc/postgres/cbo_query72.q.out index 7ebbcf51e894..bf833852b436 100644 --- a/ql/src/test/results/clientpositive/jdbc/postgres/cbo_query72.q.out +++ b/ql/src/test/results/clientpositive/jdbc/postgres/cbo_query72.q.out @@ -49,7 +49,7 @@ HiveProject(i_item_desc=[$0], w_warehouse_name=[$1], d1.d_week_seq=[$2], no_prom JdbcProject(w_warehouse_sk=[$0], w_warehouse_name=[$2]) JdbcHiveTableScan(table=[[default, warehouse]], table:alias=[warehouse]) JdbcProject(d_date_sk=[$0], d_week_seq=[$2], EXPR$0=[+(CAST($1):DOUBLE, 5.0E0)]) - JdbcFilter(condition=[AND(=($3, 2001), IS NOT NULL($0), IS NOT NULL($2), IS NOT NULL(+(CAST($1):DOUBLE, 5.0E0)))]) + JdbcFilter(condition=[AND(=($3, 2001), IS NOT NULL($0), IS NOT NULL($2), IS NOT NULL(CAST($1):DOUBLE))]) JdbcProject(d_date_sk=[$0], d_date=[$2], d_week_seq=[$4], d_year=[$6]) JdbcHiveTableScan(table=[[default, date_dim]], table:alias=[d1]) JdbcProject(d_date_sk=[$0], EXPR$0=[CAST($1):DOUBLE]) diff --git a/ql/src/test/results/clientpositive/jdbc/postgres/cbo_query88.q.out b/ql/src/test/results/clientpositive/jdbc/postgres/cbo_query88.q.out index 4a55074cd5a2..c649b279949b 100644 --- a/ql/src/test/results/clientpositive/jdbc/postgres/cbo_query88.q.out +++ b/ql/src/test/results/clientpositive/jdbc/postgres/cbo_query88.q.out @@ -25,7 +25,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ JdbcProject(ss_sold_time_sk=[$1], ss_hdemo_sk=[$5], ss_store_sk=[$7]) JdbcHiveTableScan(table=[[default, store_sales]], table:alias=[store_sales]) JdbcProject(hd_demo_sk=[$0]) - JdbcFilter(condition=[AND(<=($2, 5), OR(AND(=($1, 3), IS NOT NULL($2)), AND(=($1, 0), <=($2, 2)), AND(=($1, 1), <=($2, 3))), IN($1, 0, 1, 3), IS NOT NULL($0))]) + JdbcFilter(condition=[AND(<=($2, 5), OR(=($1, 3), AND(=($1, 0), <=($2, 2)), AND(=($1, 1), <=($2, 3))), IN($1, 0, 1, 3), IS NOT NULL($0))]) JdbcProject(hd_demo_sk=[$0], hd_dep_count=[$3], hd_vehicle_count=[$4]) JdbcHiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) JdbcProject(t_time_sk=[$0]) @@ -47,7 +47,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ JdbcProject(ss_sold_time_sk=[$1], ss_hdemo_sk=[$5], ss_store_sk=[$7]) JdbcHiveTableScan(table=[[default, store_sales]], table:alias=[store_sales]) JdbcProject(hd_demo_sk=[$0]) - JdbcFilter(condition=[AND(<=($2, 5), OR(AND(=($1, 3), IS NOT NULL($2)), AND(=($1, 0), <=($2, 2)), AND(=($1, 1), <=($2, 3))), IN($1, 0, 1, 3), IS NOT NULL($0))]) + JdbcFilter(condition=[AND(<=($2, 5), OR(=($1, 3), AND(=($1, 0), <=($2, 2)), AND(=($1, 1), <=($2, 3))), IN($1, 0, 1, 3), IS NOT NULL($0))]) JdbcProject(hd_demo_sk=[$0], hd_dep_count=[$3], hd_vehicle_count=[$4]) JdbcHiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) JdbcProject(t_time_sk=[$0]) @@ -69,7 +69,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ JdbcProject(ss_sold_time_sk=[$1], ss_hdemo_sk=[$5], ss_store_sk=[$7]) JdbcHiveTableScan(table=[[default, store_sales]], table:alias=[store_sales]) JdbcProject(hd_demo_sk=[$0]) - JdbcFilter(condition=[AND(<=($2, 5), OR(AND(=($1, 3), IS NOT NULL($2)), AND(=($1, 0), <=($2, 2)), AND(=($1, 1), <=($2, 3))), IN($1, 0, 1, 3), IS NOT NULL($0))]) + JdbcFilter(condition=[AND(<=($2, 5), OR(=($1, 3), AND(=($1, 0), <=($2, 2)), AND(=($1, 1), <=($2, 3))), IN($1, 0, 1, 3), IS NOT NULL($0))]) JdbcProject(hd_demo_sk=[$0], hd_dep_count=[$3], hd_vehicle_count=[$4]) JdbcHiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) JdbcProject(t_time_sk=[$0]) @@ -91,7 +91,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ JdbcProject(ss_sold_time_sk=[$1], ss_hdemo_sk=[$5], ss_store_sk=[$7]) JdbcHiveTableScan(table=[[default, store_sales]], table:alias=[store_sales]) JdbcProject(hd_demo_sk=[$0]) - JdbcFilter(condition=[AND(<=($2, 5), OR(AND(=($1, 3), IS NOT NULL($2)), AND(=($1, 0), <=($2, 2)), AND(=($1, 1), <=($2, 3))), IN($1, 0, 1, 3), IS NOT NULL($0))]) + JdbcFilter(condition=[AND(<=($2, 5), OR(=($1, 3), AND(=($1, 0), <=($2, 2)), AND(=($1, 1), <=($2, 3))), IN($1, 0, 1, 3), IS NOT NULL($0))]) JdbcProject(hd_demo_sk=[$0], hd_dep_count=[$3], hd_vehicle_count=[$4]) JdbcHiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) JdbcProject(t_time_sk=[$0]) @@ -113,7 +113,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ JdbcProject(ss_sold_time_sk=[$1], ss_hdemo_sk=[$5], ss_store_sk=[$7]) JdbcHiveTableScan(table=[[default, store_sales]], table:alias=[store_sales]) JdbcProject(hd_demo_sk=[$0]) - JdbcFilter(condition=[AND(<=($2, 5), OR(AND(=($1, 3), IS NOT NULL($2)), AND(=($1, 0), <=($2, 2)), AND(=($1, 1), <=($2, 3))), IN($1, 0, 1, 3), IS NOT NULL($0))]) + JdbcFilter(condition=[AND(<=($2, 5), OR(=($1, 3), AND(=($1, 0), <=($2, 2)), AND(=($1, 1), <=($2, 3))), IN($1, 0, 1, 3), IS NOT NULL($0))]) JdbcProject(hd_demo_sk=[$0], hd_dep_count=[$3], hd_vehicle_count=[$4]) JdbcHiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) JdbcProject(t_time_sk=[$0]) @@ -135,7 +135,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ JdbcProject(ss_sold_time_sk=[$1], ss_hdemo_sk=[$5], ss_store_sk=[$7]) JdbcHiveTableScan(table=[[default, store_sales]], table:alias=[store_sales]) JdbcProject(hd_demo_sk=[$0]) - JdbcFilter(condition=[AND(<=($2, 5), OR(AND(=($1, 3), IS NOT NULL($2)), AND(=($1, 0), <=($2, 2)), AND(=($1, 1), <=($2, 3))), IN($1, 0, 1, 3), IS NOT NULL($0))]) + JdbcFilter(condition=[AND(<=($2, 5), OR(=($1, 3), AND(=($1, 0), <=($2, 2)), AND(=($1, 1), <=($2, 3))), IN($1, 0, 1, 3), IS NOT NULL($0))]) JdbcProject(hd_demo_sk=[$0], hd_dep_count=[$3], hd_vehicle_count=[$4]) JdbcHiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) JdbcProject(t_time_sk=[$0]) @@ -157,7 +157,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ JdbcProject(ss_sold_time_sk=[$1], ss_hdemo_sk=[$5], ss_store_sk=[$7]) JdbcHiveTableScan(table=[[default, store_sales]], table:alias=[store_sales]) JdbcProject(hd_demo_sk=[$0]) - JdbcFilter(condition=[AND(<=($2, 5), OR(AND(=($1, 3), IS NOT NULL($2)), AND(=($1, 0), <=($2, 2)), AND(=($1, 1), <=($2, 3))), IN($1, 0, 1, 3), IS NOT NULL($0))]) + JdbcFilter(condition=[AND(<=($2, 5), OR(=($1, 3), AND(=($1, 0), <=($2, 2)), AND(=($1, 1), <=($2, 3))), IN($1, 0, 1, 3), IS NOT NULL($0))]) JdbcProject(hd_demo_sk=[$0], hd_dep_count=[$3], hd_vehicle_count=[$4]) JdbcHiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) JdbcProject(t_time_sk=[$0]) @@ -179,7 +179,7 @@ HiveProject(s1.h8_30_to_9=[$0], s2.h9_to_9_30=[$7], s3.h9_30_to_10=[$6], s4.h10_ JdbcProject(ss_sold_time_sk=[$1], ss_hdemo_sk=[$5], ss_store_sk=[$7]) JdbcHiveTableScan(table=[[default, store_sales]], table:alias=[store_sales]) JdbcProject(hd_demo_sk=[$0]) - JdbcFilter(condition=[AND(<=($2, 5), OR(AND(=($1, 3), IS NOT NULL($2)), AND(=($1, 0), <=($2, 2)), AND(=($1, 1), <=($2, 3))), IN($1, 0, 1, 3), IS NOT NULL($0))]) + JdbcFilter(condition=[AND(<=($2, 5), OR(=($1, 3), AND(=($1, 0), <=($2, 2)), AND(=($1, 1), <=($2, 3))), IN($1, 0, 1, 3), IS NOT NULL($0))]) JdbcProject(hd_demo_sk=[$0], hd_dep_count=[$3], hd_vehicle_count=[$4]) JdbcHiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) JdbcProject(t_time_sk=[$0])