Skip to content

Commit efa8120

Browse files
author
ManyaMehrotra
committed
Refactor QueryProperties feature flags to use QueryFeature enum
1 parent 1fd7161 commit efa8120

8 files changed

Lines changed: 120 additions & 109 deletions

File tree

‎itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/CheckQueryPropertiesHook.java‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.hadoop.hive.ql.hooks;
1919

2020
import org.apache.hadoop.hive.ql.QueryProperties;
21+
import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
2122
import org.apache.hadoop.hive.ql.session.SessionState;
2223
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
2324

@@ -41,13 +42,14 @@ public void run(HookContext hookContext) {
4142

4243
if (queryProps != null) {
4344
console.printError("Has Join: " + queryProps.hasJoin());
44-
console.printError("Has Group By: " + queryProps.hasGroupBy());
45-
console.printError("Has Sort By: " + queryProps.hasSortBy());
46-
console.printError("Has Order By: " + queryProps.hasOrderBy());
47-
console.printError("Has Group By After Join: " + queryProps.hasJoinFollowedByGroupBy());
48-
console.printError("Uses Script: " + queryProps.usesScript());
49-
console.printError("Has Distribute By: " + queryProps.hasDistributeBy());
50-
console.printError("Has Cluster By: " + queryProps.hasClusterBy());
45+
console.printError("Has Group By: " + queryProps.hasFeature(QueryFeature.GROUP_BY));
46+
console.printError("Has Sort By: " + queryProps.hasFeature(QueryFeature.SORT_BY));
47+
console.printError("Has Order By: " + queryProps.hasFeature(QueryFeature.ORDER_BY));
48+
console.printError("Has Group By After Join: "
49+
+ queryProps.hasFeature(QueryFeature.JOIN_FOLLOWED_BY_GROUP_BY));
50+
console.printError("Uses Script: " + queryProps.hasFeature(QueryFeature.USES_SCRIPT));
51+
console.printError("Has Distribute By: " + queryProps.hasFeature(QueryFeature.DISTRIBUTE_BY));
52+
console.printError("Has Cluster By: " + queryProps.hasFeature(QueryFeature.CLUSTER_BY));
5153
}
5254
}
5355
}

‎ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java‎

Lines changed: 67 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
package org.apache.hadoop.hive.ql;
2020

2121

22+
import java.util.EnumSet;
2223
import java.util.HashSet;
2324
import java.util.Set;
2425

@@ -56,6 +57,24 @@ public String getName() {
5657
}
5758
}
5859

60+
public enum QueryFeature {
61+
GROUP_BY,
62+
ORDER_BY,
63+
OUTER_ORDER_BY,
64+
SORT_BY,
65+
LIMIT,
66+
JOIN_FOLLOWED_BY_GROUP_BY,
67+
PTF,
68+
WINDOWING,
69+
QUALIFY,
70+
EXCEPT,
71+
INTERSECT,
72+
USES_SCRIPT,
73+
DISTRIBUTE_BY,
74+
CLUSTER_BY,
75+
MAP_GROUP_BY
76+
}
77+
5978
boolean query;
6079
boolean analyzeCommand;
6180
boolean noScanAnalyzeCommand;
@@ -64,25 +83,8 @@ public String getName() {
6483
int outerQueryLimit;
6584

6685
boolean hasJoin = false;
67-
boolean hasGroupBy = false;
68-
boolean hasOrderBy = false;
69-
boolean hasOuterOrderBy = false;
70-
boolean hasSortBy = false;
71-
boolean hasLimit = false;
72-
boolean hasJoinFollowedByGroupBy = false;
73-
boolean hasPTF = false;
74-
boolean hasWindowing = false;
75-
boolean hasQualify = false;
76-
boolean hasExcept = false;
77-
boolean hasIntersect = false;
78-
79-
// does the query have a using clause
80-
boolean usesScript = false;
81-
82-
boolean hasDistributeBy = false;
83-
boolean hasClusterBy = false;
86+
private final EnumSet<QueryFeature> features = EnumSet.noneOf(QueryFeature.class);
8487
boolean mapJoinRemoved = false;
85-
boolean hasMapGroupBy = false;
8688

8789
private boolean hasLateralViews = false;
8890
private boolean cboSupportedLateralViews = true;
@@ -196,116 +198,124 @@ public boolean isCBOSupportedLateralViews() {
196198
return cboSupportedLateralViews;
197199
}
198200

201+
public void addFeature(QueryFeature feature) {
202+
features.add(feature);
203+
}
204+
205+
public boolean hasFeature(QueryFeature feature) {
206+
return features.contains(feature);
207+
}
208+
199209
public boolean hasGroupBy() {
200-
return hasGroupBy;
210+
return hasFeature(QueryFeature.GROUP_BY);
201211
}
202212

203213
public void setHasGroupBy(boolean hasGroupBy) {
204-
this.hasGroupBy = hasGroupBy;
214+
setFeature(QueryFeature.GROUP_BY, hasGroupBy);
205215
}
206216

207217
public boolean hasOrderBy() {
208-
return hasOrderBy;
218+
return hasFeature(QueryFeature.ORDER_BY);
209219
}
210220

211221
public void setHasOrderBy(boolean hasOrderBy) {
212-
this.hasOrderBy = hasOrderBy;
222+
setFeature(QueryFeature.ORDER_BY, hasOrderBy);
213223
}
214224

215225
public boolean hasOuterOrderBy() {
216-
return hasOuterOrderBy;
226+
return hasFeature(QueryFeature.OUTER_ORDER_BY);
217227
}
218228

219229
public void setHasOuterOrderBy(boolean hasOuterOrderBy) {
220-
this.hasOuterOrderBy = hasOuterOrderBy;
230+
setFeature(QueryFeature.OUTER_ORDER_BY, hasOuterOrderBy);
221231
}
222232

223233
public boolean hasSortBy() {
224-
return hasSortBy;
234+
return hasFeature(QueryFeature.SORT_BY);
225235
}
226236

227237
public void setHasSortBy(boolean hasSortBy) {
228-
this.hasSortBy = hasSortBy;
238+
setFeature(QueryFeature.SORT_BY, hasSortBy);
229239
}
230240

231241
public void setHasLimit(boolean hasLimit) {
232-
this.hasLimit = hasLimit;
242+
setFeature(QueryFeature.LIMIT, hasLimit);
233243
}
234244

235245
public boolean hasLimit() {
236-
return hasLimit;
246+
return hasFeature(QueryFeature.LIMIT);
237247
}
238248

239249
public boolean hasJoinFollowedByGroupBy() {
240-
return hasJoinFollowedByGroupBy;
250+
return hasFeature(QueryFeature.JOIN_FOLLOWED_BY_GROUP_BY);
241251
}
242252

243253
public void setHasJoinFollowedByGroupBy(boolean hasJoinFollowedByGroupBy) {
244-
this.hasJoinFollowedByGroupBy = hasJoinFollowedByGroupBy;
254+
setFeature(QueryFeature.JOIN_FOLLOWED_BY_GROUP_BY, hasJoinFollowedByGroupBy);
245255
}
246256

247257
public boolean usesScript() {
248-
return usesScript;
258+
return hasFeature(QueryFeature.USES_SCRIPT);
249259
}
250260

251261
public void setUsesScript(boolean usesScript) {
252-
this.usesScript = usesScript;
262+
setFeature(QueryFeature.USES_SCRIPT, usesScript);
253263
}
254264

255265
public boolean hasDistributeBy() {
256-
return hasDistributeBy;
266+
return hasFeature(QueryFeature.DISTRIBUTE_BY);
257267
}
258268

259269
public void setHasDistributeBy(boolean hasDistributeBy) {
260-
this.hasDistributeBy = hasDistributeBy;
270+
setFeature(QueryFeature.DISTRIBUTE_BY, hasDistributeBy);
261271
}
262272

263273
public boolean hasClusterBy() {
264-
return hasClusterBy;
274+
return hasFeature(QueryFeature.CLUSTER_BY);
265275
}
266276

267277
public void setHasClusterBy(boolean hasClusterBy) {
268-
this.hasClusterBy = hasClusterBy;
278+
setFeature(QueryFeature.CLUSTER_BY, hasClusterBy);
269279
}
270280

271281
public boolean hasPTF() {
272-
return hasPTF;
282+
return hasFeature(QueryFeature.PTF);
273283
}
274284

275285
public void setHasPTF(boolean hasPTF) {
276-
this.hasPTF = hasPTF;
286+
setFeature(QueryFeature.PTF, hasPTF);
277287
}
278288

279289
public boolean hasWindowing() {
280-
return hasWindowing;
290+
return hasFeature(QueryFeature.WINDOWING);
281291
}
282292

283293
public void setHasWindowing(boolean hasWindowing) {
284-
this.hasWindowing = hasWindowing;
294+
setFeature(QueryFeature.WINDOWING, hasWindowing);
285295
}
286296

287297
public boolean hasQualify() {
288-
return hasQualify;
298+
return hasFeature(QueryFeature.QUALIFY);
289299
}
290300

291301
public void setHasQualify(boolean hasQualify) {
292-
this.hasQualify = hasQualify;
302+
setFeature(QueryFeature.QUALIFY, hasQualify);
293303
}
294304

295305
public boolean hasExcept() {
296-
return hasExcept;
306+
return hasFeature(QueryFeature.EXCEPT);
297307
}
298308

299309
public void setHasExcept(boolean hasExcept) {
300-
this.hasExcept = hasExcept;
310+
setFeature(QueryFeature.EXCEPT, hasExcept);
301311
}
302312

303313
public boolean hasIntersect() {
304-
return hasIntersect;
314+
return hasFeature(QueryFeature.INTERSECT);
305315
}
306316

307317
public void setHasIntersect(boolean hasIntersect) {
308-
this.hasIntersect = hasIntersect;
318+
setFeature(QueryFeature.INTERSECT, hasIntersect);
309319
}
310320

311321
public boolean isMapJoinRemoved() {
@@ -317,11 +327,11 @@ public void setMapJoinRemoved(boolean mapJoinRemoved) {
317327
}
318328

319329
public boolean isHasMapGroupBy() {
320-
return hasMapGroupBy;
330+
return hasFeature(QueryFeature.MAP_GROUP_BY);
321331
}
322332

323333
public void setHasMapGroupBy(boolean hasMapGroupBy) {
324-
this.hasMapGroupBy = hasMapGroupBy;
334+
setFeature(QueryFeature.MAP_GROUP_BY, hasMapGroupBy);
325335
}
326336

327337
public boolean hasMultiDestQuery() {
@@ -386,24 +396,8 @@ public void clear() {
386396
isMaterializedView = false;
387397

388398
hasJoin = false;
389-
hasGroupBy = false;
390-
hasOrderBy = false;
391-
hasOuterOrderBy = false;
392-
hasSortBy = false;
393-
hasJoinFollowedByGroupBy = false;
394-
hasPTF = false;
395-
hasWindowing = false;
396-
hasQualify = false;
397-
hasExcept = false;
398-
hasIntersect = false;
399-
400-
// does the query have a using clause
401-
usesScript = false;
402-
403-
hasDistributeBy = false;
404-
hasClusterBy = false;
399+
features.clear();
405400
mapJoinRemoved = false;
406-
hasMapGroupBy = false;
407401

408402
noOfJoins = 0;
409403
noOfOuterJoins = 0;
@@ -413,4 +407,12 @@ public void clear() {
413407

414408
usedTables.clear();
415409
}
410+
411+
private void setFeature(QueryFeature feature, boolean enabled) {
412+
if (enabled) {
413+
addFeature(feature);
414+
} else {
415+
features.remove(feature);
416+
}
417+
}
416418
}

‎ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.slf4j.LoggerFactory;
3333
import org.apache.hadoop.hive.conf.HiveConf;
3434
import org.apache.hadoop.hive.metastore.api.FieldSchema;
35+
import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
3536
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
3637
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
3738
import org.apache.hadoop.hive.ql.exec.Operator;
@@ -212,7 +213,7 @@ else if (!HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_GROUPBY_SKEW)) {
212213
convertGroupByMapSideSortedGroupBy(hiveConf, groupByOp, depth);
213214
}
214215
else if (optimizeDistincts && !HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED)) {
215-
pGraphContext.getQueryProperties().setHasMapGroupBy(true);
216+
pGraphContext.getQueryProperties().addFeature(QueryFeature.MAP_GROUP_BY);
216217
ReduceSinkOperator reduceSinkOp =
217218
(ReduceSinkOperator)groupByOp.getChildOperators().get(0);
218219
GroupByDesc childGroupByDesc =
@@ -514,7 +515,7 @@ private GroupByOptimizerSortMatch matchBucketSortCols(
514515
// The operators specified by depth and removed from the tree.
515516
protected void convertGroupByMapSideSortedGroupBy(
516517
HiveConf conf, GroupByOperator groupByOp, int depth) {
517-
pGraphContext.getQueryProperties().setHasMapGroupBy(true);
518+
pGraphContext.getQueryProperties().addFeature(QueryFeature.MAP_GROUP_BY);
518519

519520
if (removeChildren(groupByOp, depth)) {
520521
// Use bucketized hive input format - that makes sure that one mapper reads the entire file

‎ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import org.apache.hadoop.hive.conf.HiveConf;
3939
import org.apache.hadoop.hive.metastore.api.FieldSchema;
4040
import org.apache.hadoop.hive.metastore.api.Order;
41+
import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
4142
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
4243
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
4344
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
@@ -732,7 +733,7 @@ public ReduceSinkOperator getReduceSinkOp(List<Integer> partitionPositions, List
732733
// should honor the ordering of records provided by ORDER BY in SELECT statement
733734
ReduceSinkOperator parentRSOp = OperatorUtils.findSingleOperatorUpstream(parent,
734735
ReduceSinkOperator.class);
735-
if (parentRSOp != null && parseCtx.getQueryProperties().hasOuterOrderBy()) {
736+
if (parentRSOp != null && parseCtx.getQueryProperties().hasFeature(QueryFeature.OUTER_ORDER_BY)) {
736737
String parentRSOpOrder = parentRSOp.getConf().getOrder();
737738
String parentRSOpNullOrder = parentRSOp.getConf().getNullOrder();
738739
if (parentRSOpOrder != null && !parentRSOpOrder.isEmpty() && sortPositions.isEmpty()) {

‎ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@
141141
import org.apache.hadoop.hive.ql.Context;
142142
import org.apache.hadoop.hive.ql.ErrorMsg;
143143
import org.apache.hadoop.hive.ql.QueryProperties;
144+
import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
144145
import org.apache.hadoop.hive.ql.QueryState;
145146
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
146147
import org.apache.hadoop.hive.ql.exec.FunctionInfo;
@@ -689,9 +690,9 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept
689690
this.ctx.setCboInfo(cboMsg);
690691

691692
// Determine if we should re-throw the exception OR if we try to mark the query to retry as non-CBO.
692-
final boolean requiresCBO = queryProperties.hasQualify()
693-
|| queryProperties.hasExcept()
694-
|| queryProperties.hasIntersect();
693+
final boolean requiresCBO = queryProperties.hasFeature(QueryFeature.QUALIFY)
694+
|| queryProperties.hasFeature(QueryFeature.EXCEPT)
695+
|| queryProperties.hasFeature(QueryFeature.INTERSECT);
695696
if (requiresCBO || fallbackStrategy.isFatal(e)) {
696697
if (e instanceof RuntimeException || e instanceof SemanticException) {
697698
// These types of exceptions do not need wrapped
@@ -1000,13 +1001,13 @@ private static String canHandleQbForCbo(QueryProperties queryProperties,
10001001
HiveConf conf, boolean topLevelQB) {
10011002
List<String> reasons = new ArrayList<>();
10021003
// Not ok to run CBO, build error message.
1003-
if (queryProperties.hasSortBy() && queryProperties.hasLimit()) {
1004+
if (queryProperties.hasFeature(QueryFeature.SORT_BY) && queryProperties.hasFeature(QueryFeature.LIMIT)) {
10041005
reasons.add("has sort by with limit");
10051006
}
1006-
if (queryProperties.hasPTF()) {
1007+
if (queryProperties.hasFeature(QueryFeature.PTF)) {
10071008
reasons.add("has PTF");
10081009
}
1009-
if (queryProperties.usesScript()) {
1010+
if (queryProperties.hasFeature(QueryFeature.USES_SCRIPT)) {
10101011
reasons.add("uses scripts");
10111012
}
10121013
if (!queryProperties.isCBOSupportedLateralViews()) {
@@ -1024,7 +1025,7 @@ private static EnumSet<ExtendedCBOProfile> obtainCBOProfiles(QueryProperties que
10241025
profilesCBO.add(ExtendedCBOProfile.JOIN_REORDERING);
10251026
}
10261027
// If the query contains windowing processing
1027-
if (queryProperties.hasWindowing()) {
1028+
if (queryProperties.hasFeature(QueryFeature.WINDOWING)) {
10281029
profilesCBO.add(ExtendedCBOProfile.WINDOWING_POSTPROCESSING);
10291030
}
10301031
return profilesCBO;

0 commit comments

Comments
 (0)