Skip to content

Commit

Permalink
improved big query sql parser
Browse files Browse the repository at this point in the history
  • Loading branch information
@wenshao
wenshao committed Oct 29, 2024
1 parent cec53c0 commit 92bd9c4
Show file tree
Hide file tree
Showing 13 changed files with 291 additions and 78 deletions.
10 changes: 10 additions & 0 deletions core/src/main/java/com/alibaba/druid/sql/ast/expr/SQLAllColumnExpr.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.alibaba.druid.sql.ast.statement.SQLTableSource;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

Expand All @@ -28,11 +29,16 @@ public final class SQLAllColumnExpr extends SQLExprImpl {

private SQLExpr owner;
private List<SQLExpr> except;
private final List<SQLAliasedExpr> replace = new ArrayList<SQLAliasedExpr>();

public SQLAllColumnExpr() {
}

public void output(StringBuilder buf) {
if (owner != null) {
owner.output(buf);
buf.append('.');
}
buf.append('*');
}

Expand All @@ -51,6 +57,10 @@ public List<SQLExpr> getExcept() {
return except;
}

public List<SQLAliasedExpr> getReplace() {
return replace;
}

public void setExcept(List<SQLExpr> except) {
this.except = except;
}
Expand Down
11 changes: 11 additions & 0 deletions core/src/main/java/com/alibaba/druid/sql/ast/expr/SQLStructExpr.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,25 @@

import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLExprImpl;
import com.alibaba.druid.sql.ast.SQLStructDataType;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;

import java.util.ArrayList;
import java.util.List;

public class SQLStructExpr extends SQLExprImpl {
private SQLStructDataType dataType;
private final List<SQLAliasedExpr> items = new ArrayList<>();

public SQLStructDataType getDataType() {
return dataType;
}

public void setDataType(SQLStructDataType x) {
x.setParent(this);
this.dataType = x;
}

public void addItem(SQLAliasedExpr item) {
item.setParent(this);
items.add(item);
Expand Down
38 changes: 18 additions & 20 deletions core/src/main/java/com/alibaba/druid/sql/dialect/bigquery/parser/BigQueryExprParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
import com.alibaba.druid.util.FnvHash;

import java.util.Arrays;
import java.util.List;

import static com.alibaba.druid.util.FnvHash.fnv1a_64_lower;

Expand All @@ -29,6 +28,7 @@ public class BigQueryExprParser extends SQLExprParser {
"BIT_XOR",
"COUNT",
"COUNTIF",
"FIRST_VALUE",
"GROUPING",
"LOGICAL_AND",
"LOGICAL_OR",
Expand Down Expand Up @@ -107,25 +107,6 @@ private SQLStructExpr struct() {
return structExpr;
}

protected void aliasedItems(List<SQLAliasedExpr> items, SQLObject parent) {
while (true) {
SQLExpr expr = expr();
String alias = as();

SQLAliasedExpr aliasedExpr = new SQLAliasedExpr(expr, alias);
aliasedExpr.setParent(parent);
items.add(aliasedExpr);

if (lexer.nextIfComma()) {
if (lexer.token() == Token.FROM || lexer.token() == Token.RPAREN) {
break;
}
continue;
}
break;
}
}

public SQLColumnDefinition parseColumnRest(SQLColumnDefinition column) {
if (lexer.nextIfIdentifier(FnvHash.Constants.OPTIONS)) {
parseAssignItem(column.getColProperties(), column);
Expand All @@ -136,7 +117,10 @@ public SQLColumnDefinition parseColumnRest(SQLColumnDefinition column) {

protected SQLStructDataType parseDataTypeStruct() {
acceptIdentifier("STRUCT");
return parseDataTypeStruct0();
}

protected SQLStructDataType parseDataTypeStruct0() {
SQLStructDataType struct = new SQLStructDataType(dbType);
accept(Token.LT);
for (; ; ) {
Expand Down Expand Up @@ -335,4 +319,18 @@ protected SQLExpr dotRest(SQLExpr expr) {
public SQLSelectParser createSelectParser() {
return new BigQuerySelectParser(this, null);
}

public SQLExpr exprRest(SQLExpr expr) {
if (lexer.token() == Token.LT && expr instanceof SQLIdentifierExpr && ((SQLIdentifierExpr) expr).nameEquals("STRUCT")) {
SQLStructExpr structExpr = new SQLStructExpr();
structExpr.setDataType(
parseDataTypeStruct0()
);
accept(Token.LPAREN);
aliasedItems(structExpr.getItems(), structExpr);
accept(Token.RPAREN);
expr = structExpr;
}
return super.exprRest(expr);
}
}
121 changes: 121 additions & 0 deletions core/src/main/java/com/alibaba/druid/sql/dialect/bigquery/parser/BigQueryLexer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

import static com.alibaba.druid.sql.parser.DialectFeature.LexerFeature.ScanSubAsIdentifier;
import static com.alibaba.druid.sql.parser.DialectFeature.ParserFeature.*;
import static com.alibaba.druid.sql.parser.Token.LITERAL_CHARS;

public class BigQueryLexer extends Lexer {
@Override
Expand Down Expand Up @@ -38,6 +39,7 @@ protected Keywords loadKeywords() {
map.put("DISTINCT", Token.DISTINCT);
map.put("DELETE", Token.DELETE);
map.put("DROP", Token.DROP);
map.put("DO", Token.DO);
map.put("ELSE", Token.ELSE);
map.put("END", Token.END);
map.put("ESCAPE", Token.ESCAPE);
Expand Down Expand Up @@ -100,6 +102,7 @@ protected Keywords loadKeywords() {
map.put("WHERE", Token.WHERE);
map.put("WINDOW", Token.WINDOW);
map.put("WITH", Token.WITH);
map.put("WHILE", Token.WHILE);
map.put("VIEW", Token.VIEW);

return new Keywords(map);
Expand Down Expand Up @@ -162,4 +165,122 @@ public void scanComment() {
throw new IllegalStateException();
}
}

@Override
protected void scanString() {
{
boolean hasSpecial = false;
int startIndex = pos + 1;
int endIndex = -1; // text.indexOf('\'', startIndex);
for (int i = startIndex; i < text.length(); ++i) {
final char ch = text.charAt(i);
if (ch == '\\') {
hasSpecial = true;
continue;
}
if (ch == '\'') {
endIndex = i;
break;
}
}

if (endIndex == -1) {
throw new ParserException("unclosed str. " + info());
}

String stringVal;
if (token == Token.AS) {
stringVal = text.substring(pos, endIndex + 1);
} else {
if (startIndex == endIndex) {
stringVal = "";
} else {
stringVal = text.substring(startIndex, endIndex);
}
}
// hasSpecial = stringVal.indexOf('\\') != -1;

if (!hasSpecial) {
this.stringVal = stringVal;
int pos = endIndex + 1;
char ch = charAt(pos);
if (ch != '\'') {
this.pos = pos;
this.ch = ch;
token = LITERAL_CHARS;
return;
}
}
}

mark = pos;
boolean hasSpecial = false;
for (; ; ) {
if (isEOF()) {
lexError("unclosed.str.lit");
return;
}

ch = charAt(++pos);

if (ch == '\\') {
scanChar();
if (!hasSpecial) {
initBuff(bufPos);
arraycopy(mark + 1, buf, 0, bufPos);
hasSpecial = true;
}

switch (ch) {
case '\'':
putChar('\'');
break;
case '"':
putChar('"');
break;
case '\\':
putChar('\\');
break;
default:
putChar('\\');
putChar(ch);
break;
}

continue;
}
if (ch == '\'') {
scanChar();
if (ch != '\'') {
token = LITERAL_CHARS;
break;
} else {
if (!hasSpecial) {
initBuff(bufPos);
arraycopy(mark + 1, buf, 0, bufPos);
hasSpecial = true;
}
putChar('\'');
continue;
}
}

if (!hasSpecial) {
bufPos++;
continue;
}

if (bufPos == buf.length) {
putChar(ch);
} else {
buf[bufPos++] = ch;
}
}

if (!hasSpecial) {
stringVal = subString(mark + 1, bufPos);
} else {
stringVal = new String(buf, 0, bufPos);
}
}
}
27 changes: 0 additions & 27 deletions core/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlStatementParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8737,33 +8737,6 @@ public SQLIfStatement parseIf() {
return stmt;
}

/**
* parse while statement
*
* @return MySqlWhileStatement
*/
public SQLWhileStatement parseWhile() {
accept(Token.WHILE);
SQLWhileStatement stmt = new SQLWhileStatement();

stmt.setCondition(this.exprParser.expr());

accept(Token.DO);

this.parseStatementList(stmt.getStatements(), -1, stmt);

accept(Token.END);

accept(Token.WHILE);

accept(SEMI);

stmt.setAfterSemi(true);

return stmt;

}

/**
* parse while statement with label
*
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleStatementParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -959,7 +959,7 @@ public SQLStatement parseReturn() {
return stmt;
}

public SQLStatement parseWhile() {
public SQLWhileStatement parseWhile() {
accept(Token.WHILE);

SQLWhileStatement stmt = new SQLWhileStatement();
Expand Down
52 changes: 36 additions & 16 deletions core/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1995,24 +1995,44 @@ protected SQLExpr methodRest(SQLExpr expr, boolean acceptLPAREN) {
protected String doRestSpecific(SQLExpr expr) {
return null;
}
protected SQLExpr dotRest(SQLExpr expr) {
if (lexer.token == Token.STAR) {
lexer.nextToken();
if (lexer.token == Token.EXCEPT) {
SQLAllColumnExpr allColumnExpr = new SQLAllColumnExpr();
allColumnExpr.setOwner(expr);

lexer.nextToken();
accept(Token.LPAREN);
List<SQLExpr> except = new ArrayList<>();
this.exprList(except, allColumnExpr);
allColumnExpr.setExcept(except);
accept(Token.RPAREN);
protected void aliasedItems(List<SQLAliasedExpr> items, SQLObject parent) {
while (true) {
SQLExpr expr = expr();
String alias = as();

expr = allColumnExpr;
} else {
expr = new SQLPropertyExpr(expr, "*");
SQLAliasedExpr aliasedExpr = new SQLAliasedExpr(expr, alias);
aliasedExpr.setParent(parent);
items.add(aliasedExpr);

if (lexer.nextIfComma()) {
if (lexer.token() == Token.FROM || lexer.token() == Token.RPAREN) {
break;
}
continue;
}
break;
}
}

protected SQLExpr dotRest(SQLExpr expr) {
if (lexer.nextIf(STAR)) {
expr = new SQLPropertyExpr(expr, "*");
// SQLAllColumnExpr allColumnExpr = new SQLAllColumnExpr();
// allColumnExpr.setOwner(expr);
// if (lexer.nextIf(EXCEPT)) {
// accept(Token.LPAREN);
// List<SQLExpr> except = new ArrayList<>();
// this.exprList(except, allColumnExpr);
// allColumnExpr.setExcept(except);
// accept(Token.RPAREN);
// }
// if (lexer.nextIf(REPLACE)) {
// accept(Token.LPAREN);
// this.aliasedItems(allColumnExpr.getReplace(), allColumnExpr);
// accept(Token.RPAREN);
// }
// expr = allColumnExpr;
} else {
String name;
long hash_lower = 0L;
Expand Down Expand Up @@ -5698,7 +5718,7 @@ public SQLSelectItem parseSelectItem() {
}
} else if (FnvHash.Constants.DATE == hash_lower
&& lexer.stringVal().charAt(0) != '`'
&& lexer.token == Token.LITERAL_CHARS
&& (lexer.token == Token.LITERAL_CHARS || lexer.token == LITERAL_ALIAS)
&& (dialectFeatureEnabled(SQLDateExpr))
) {
String literal = lexer.stringVal();
Expand Down
Loading

0 comments on commit 92bd9c4

Please sign in to comment.