Merge dce12677d0 into c8b12f0701

* add dbtype adb_mysql

* Presto/Trino parser fix

* add CTE visit for insertNode

* add keyword config for spark/hive

* support synapse parser

* support synapse parser

* lexer fix: comment character missing

* lexer fix: comment character missing

* fix quote parsing in columns

core/src/main/java/com/alibaba/druid/pool/ValidConnectionCheckerAdapter.java

@@ -66,7 +66,7 @@ public class ValidConnectionCheckerAdapter implements ValidConnectionChecker {
                 stmt.setQueryTimeout(validationQueryTimeout);
             }
             rs = stmt.executeQuery(query);
-            return rs.next();
+            return true;
         } finally {
             JdbcUtils.close(rs);
             if (!isDruidStatementConnection) {

core/src/main/java/com/alibaba/druid/sql/ast/expr/SQLBinaryOperator.java

@@ -99,6 +99,7 @@ public enum SQLBinaryOperator {
     BooleanXor("XOR", 150),
     BooleanOr("OR", 160),
     Assignment(":=", 169),
+    Blank("", 170),
 
     PG_And("&&", 140),
     PG_ST_DISTANCE("<->", 20);

core/src/main/java/com/alibaba/druid/sql/ast/expr/SQLVariantRefExpr.java

@@ -29,16 +29,30 @@ public class SQLVariantRefExpr extends SQLExprImpl {
 
     private boolean global;
     private boolean session;
+    private boolean templateParameter;
+    private boolean hasPrefixComma;
 
     private int index = -1;
 
     public SQLVariantRefExpr(String name) {
         this.name = name;
+        if (name.startsWith("${") && name.endsWith("}")) {
+            this.templateParameter = true;
+        } else {
+            this.templateParameter = false;
+        }
+        this.hasPrefixComma = true;
     }
 
     public SQLVariantRefExpr(String name, SQLObject parent) {
         this.name = name;
         this.parent = parent;
+        if (name.startsWith("${") && name.endsWith("}")) {
+            this.templateParameter = true;
+        } else {
+            this.templateParameter = false;
+        }
+        this.hasPrefixComma = true;
     }
 
     public SQLVariantRefExpr(String name, boolean global) {
@@ -70,6 +84,22 @@ public class SQLVariantRefExpr extends SQLExprImpl {
         this.name = name;
     }
 
+    public boolean isTemplateParameter() {
+        return templateParameter;
+    }
+
+    public void setTemplateParameter(boolean templateParameter) {
+        this.templateParameter = templateParameter;
+    }
+
+    public boolean isHasPrefixComma() {
+        return hasPrefixComma;
+    }
+
+    public void setHasPrefixComma(boolean hasPrefixComma) {
+        this.hasPrefixComma = hasPrefixComma;
+    }
+
     public void output(StringBuilder buf) {
         buf.append(this.name);
     }

core/src/main/java/com/alibaba/druid/sql/parser/Lexer.java

@@ -2278,7 +2278,7 @@ public class Lexer {
         if (ch != ':' && ch != '#' && ch != '$' && !(ch == '@' && dialectFeatureEnabled(ScanVariableAt))) {
             throw new ParserException("illegal variable. " + info());
         }
-
+        boolean templateParameter = false;
         mark = pos;
         bufPos = 1;
         char ch;
@@ -2296,13 +2296,14 @@ public class Lexer {
             boolean ident = false;
             for (; ; ) {
                 ch = charAt(++pos);
-                if (isEOF() || ch == ';' || ch == '；' || ch == '\r' || ch == '\n') {
+                if (isEOF() || (templateParameter && (ch == ';' || ch == '；' || ch == '\r'))) {
                     pos--;
                     bufPos--;
                     break;
                 }
 
                 if (ch == '}' && !ident) {
+                    templateParameter = false;
                     if (isIdentifierChar(charAt(pos + 1))) {
                         bufPos++;
                         ident = true;
@@ -2313,6 +2314,7 @@ public class Lexer {
 
                 if (ident && ch == '$') {
                     if (charAt(pos + 1) == '{') {
+                        templateParameter = true;
                         bufPos++;
                         ident = false;
                         continue;
@@ -2323,7 +2325,7 @@ public class Lexer {
                     if (isWhitespace(ch)) {
                         pos--;
                         break;
-                    } else if (ch == ',' || ch == ')' || ch == '(' || ch == ';') {
+                    } else if (ch == ',' || ch == ')' || ch == '(') {
                         pos--;
                         break;
                     }

core/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java

@@ -2927,8 +2927,7 @@ public class SQLExprParser extends SQLParser {
         SQLSelectOrderByItem item = parseSelectOrderByItem();
         item.setParent(parent);
         items.add(item);
-        while (lexer.token == Token.COMMA) {
-            lexer.nextToken();
+        while (lexer.nextIf(Token.COMMA) || (item.getExpr() instanceof SQLVariantRefExpr && lexer.token == IDENTIFIER)) {
             item = parseSelectOrderByItem();
             item.setParent(parent);
             items.add(item);
@@ -3532,6 +3531,14 @@ public class SQLExprParser extends SQLParser {
                 SQLBinaryOperator operator = andRestGetAndOperator();
 
                 expr = new SQLBinaryOpExpr(expr, operator, rightExp, dbType);
+            } else if (token == Token.VARIANT) {
+                String value = lexer.stringVal();
+                lexer.nextToken();
+                SQLExpr variantExpr = new SQLVariantRefExpr(value);
+                if (lexer.token == Token.IN) {
+                    variantExpr = inRest(variantExpr);
+                }
+                expr = new SQLBinaryOpExpr(expr, SQLBinaryOperator.Blank, variantExpr, dbType);
             } else {
                 break;
             }
@@ -3619,6 +3626,14 @@ public class SQLExprParser extends SQLParser {
                 SQLBinaryOperator op = orRestGetOrOperator();
 
                 expr = new SQLBinaryOpExpr(expr, op, rightExp, dbType);
+            } else if (lexer.token == Token.VARIANT) {
+                String value = lexer.stringVal();
+                lexer.nextToken();
+                SQLExpr variantExpr = new SQLVariantRefExpr(value);
+                if (lexer.token == Token.IN) {
+                    variantExpr = inRest(variantExpr);
+                }
+                expr = new SQLBinaryOpExpr(expr, SQLBinaryOperator.Blank, variantExpr, dbType);
             } else {
                 break;
             }
@@ -3977,6 +3992,11 @@ public class SQLExprParser extends SQLParser {
                     return expr;
                 }
                 break;
+            case VARIANT:
+                rightExp = new SQLVariantRefExpr(lexer.stringVal);
+                expr = new SQLBinaryOpExpr(expr, SQLBinaryOperator.Blank, rightExp, dbType);
+                lexer.nextToken();
+                return expr;
             default:
                 return expr;
         }
@@ -5946,46 +5966,47 @@ public class SQLExprParser extends SQLParser {
 
         String alias;
         List<String> aliasList = null;
-        switch (lexer.token) {
-            case FULL:
-            case TABLESPACE:
-                alias = lexer.stringVal();
-                lexer.nextToken();
-                break;
-            case AS:
-                lexer.nextTokenAlias();
-                if (lexer.token == Token.LITERAL_INT) {
-                    alias = '"' + lexer.stringVal() + '"';
+        if (expr instanceof SQLVariantRefExpr && ((SQLVariantRefExpr) expr).isTemplateParameter() && lexer.token != AS) {
+            alias = null;
+        } else {
+            switch (lexer.token) {
+                case FULL:
+                case TABLESPACE:
+                    alias = lexer.stringVal();
                     lexer.nextToken();
-                } else if (lexer.token == Token.LPAREN) {
-                    lexer.nextToken();
-                    aliasList = new ArrayList<String>();
-
-                    for (; ; ) {
-                        String stringVal = lexer.stringVal();
+                    break;
+                case AS:
+                    lexer.nextTokenAlias();
+                    if (lexer.token == Token.LITERAL_INT) {
+                        alias = '"' + lexer.stringVal() + '"';
                         lexer.nextToken();
-
-                        aliasList.add(stringVal);
-
-                        if (lexer.token() == Token.COMMA) {
+                    } else if (lexer.token == Token.LPAREN) {
+                        lexer.nextToken();
+                        aliasList = new ArrayList<String>();
+                        for (; ; ) {
+                            String stringVal = lexer.stringVal();
                             lexer.nextToken();
-                            continue;
+                            aliasList.add(stringVal);
+                            if (lexer.token() == Token.COMMA) {
+                                lexer.nextToken();
+                                continue;
+                            }
+                            break;
                         }
-                        break;
-                    }
-                    accept(Token.RPAREN);
+                        accept(Token.RPAREN);
 
+                        alias = null;
+                    } else {
+                        alias = alias();
+                    }
+                    break;
+                case EOF:
                     alias = null;
-                } else {
-                    alias = alias();
-                }
-                break;
-            case EOF:
-                alias = null;
-                break;
-            default:
-                alias = as();
-                break;
+                    break;
+                default:
+                    alias = as();
+                    break;
+            }
         }
 
         if (alias == null && isEnabled(SQLParserFeature.SelectItemGenerateAlias)

core/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java

@@ -756,6 +756,7 @@ public class SQLSelectParser extends SQLParser {
                     case BANGEQ:
                     case LIKE:
                     case NOT:
+                    case VARIANT:
                         where = this.exprParser.relationalRest(where);
                         break;
                     default:
@@ -1122,11 +1123,16 @@ public class SQLSelectParser extends SQLParser {
 
     protected void parseSelectList(SQLSelectQueryBlock queryBlock) {
         final List<SQLSelectItem> selectList = queryBlock.getSelectList();
+        boolean hasComma = true;
         for (; ; ) {
             final SQLSelectItem selectItem = this.exprParser.parseSelectItem();
             selectList.add(selectItem);
             selectItem.setParent(queryBlock);
-
+            if (!hasComma && selectItem.getExpr() instanceof SQLVariantRefExpr) {
+                SQLVariantRefExpr sqlVariantRefExpr = (SQLVariantRefExpr) selectItem.getExpr();
+                sqlVariantRefExpr.setHasPrefixComma(false);
+                hasComma = true;
+            }
             //https://github.com/alibaba/druid/issues/5708
             if (lexer.hasComment()
                     && lexer.isKeepComments()
@@ -1136,7 +1142,14 @@ public class SQLSelectParser extends SQLParser {
             }
 
             if (lexer.token != Token.COMMA) {
-                break;
+                if (lexer.token == Token.VARIANT) {
+                    hasComma = false;
+                    continue;
+                } else if (selectItem.getExpr() instanceof SQLVariantRefExpr && ((SQLVariantRefExpr) selectItem.getExpr()).isTemplateParameter() && lexer.token != Token.FROM) {
+                    continue;
+                } else {
+                    break;
+                }
             }
 
             int line = lexer.line;

core/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java

@@ -222,11 +222,11 @@ public class SQLStatementParser extends SQLParser {
                 case SELECT: {
                     MySqlHintStatement hintStatement = null;
                     if (i == 1
-                            && statementList.size() > 0
+                            && !statementList.isEmpty()
                             && statementList.get(statementList.size() - i) instanceof MySqlHintStatement) {
                         hintStatement = (MySqlHintStatement) statementList.get(statementList.size() - i);
                     } else if (i > 0 && dialectFeatureEnabled(ParseStatementListSelectUnsupportedSyntax) && !semi
-                            && !(statementList.size() > 0 && statementList.get(statementList.size() - i).isAfterSemi())
+                            && !(!statementList.isEmpty() && statementList.size() - i >= 0 && statementList.get(statementList.size() - i).isAfterSemi())
                     ) {
                         throw new ParserException("syntax error. " + lexer.info());
                     }

core/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java

@@ -586,8 +586,11 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
                     lineItemCount = 0;
                     println();
                 }
-
-                print0(", ");
+                if (selectItem.getExpr() instanceof SQLVariantRefExpr && !((SQLVariantRefExpr) selectItem.getExpr()).isHasPrefixComma()) {
+                    print0(" ");
+                } else {
+                    print0(", ");
+                }
             }
 
             if (selectItem.getClass() == SQLSelectItem.class) {
@@ -1048,7 +1051,7 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
                         && ((SQLIdentifierExpr) right).getName().equalsIgnoreCase("NOTFOUND")) {
                     printOpSpace = false;
                 }
-                if (printOpSpace) {
+                if (printOpSpace && operator != SQLBinaryOperator.Blank) {
                     print(' ');
                 }
             }
@@ -1569,7 +1572,12 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
         if (x.isParenthesized()) {
             print('(');
         }
-        printName0(x.getName());
+        String replacedName = x.getName();
+        if (x.getParent() instanceof SQLBinaryOpExpr
+        || x.getParent() instanceof SQLSelectItem) {
+            replacedName = replaceQuota(x.getName());
+        }
+        printName0(replacedName);
         if (x.getCollate() != null) {
             String collate = x.getCollate();
             print(" COLLATE ");

core/src/test/resources/bvt/parser/clickhouse/0.txt

@@ -1,3 +1,65 @@
+select c from b where c=1 ${if(len(a)=0,'and 1=1',"and a>0")} ${if(len(a)=0,'and 1=1',"and a>0")}
+--------------------
+SELECT c
+FROM b
+WHERE c = 1 ${if(len(a)=0,'and 1=1',"and a>0")} ${if(len(a)=0,'and 1=1',"and a>0")}
+------------------------------------------------------------------------------------------------------------------------
+select  ${if(len(a)=0,' 1=1 ',"c")}， ${if(len(a)=0,' 1=1 ',"c")} from b
+--------------------
+SELECT ${if(len(a)=0,' 1=1 ',"c")}, ${if(len(a)=0,' 1=1 ',"c")}
+FROM b
+------------------------------------------------------------------------------------------------------------------------
+select a from test where 1=1 ${if(a=1, "and b","and c")} in d
+--------------------
+SELECT a
+FROM test
+WHERE 1 = 1 ${if(a=1, "and b","and c")} IN (d)
+------------------------------------------------------------------------------------------------------------------------ 
+select a from test where 1=1
+				and (
+					a.report_item_code in ('${gFinReportItem}')  or a.report_item_code ='B002'
+					${if(find('E069',gFinReportItem)>0," or (c.yn_offset_report = 'Y' and a.report_item_code ='E06802') "," ")}
+				)
+--------------------
+SELECT a
+FROM test
+WHERE 1 = 1
+	AND (a.report_item_code IN ('${gFinReportItem}')
+	OR a.report_item_code = 'B002' ${if(find('E069',gFinReportItem)>0," or (c.yn_offset_report = 'Y' and a.report_item_code ='E06802') "," ")})
+------------------------------------------------------------------------------------------------------------------------ 
+select  ${if(len(a)=0,' 1=1 ',"c")}, ${if(len(a)=0,' 1=1 ',"c")} from b
+--------------------
+SELECT ${if(len(a)=0,' 1=1 ',"c")}, ${if(len(a)=0,' 1=1 ',"c")}
+FROM b
+------------------------------------------------------------------------------------------------------------------------ 
+        select b.merge_region_code as region_code,b.merge_region_name as region_name,b.govern_region_area_union as region_area_subtotal
+		      ${if(gRegionAreaLevel='合并营运区',",b.govern_region_area_union as region_area_code,b.govern_region_area_union as region_area_name",",b.region_area_code as region_area_code,b.region_area_name as region_area_name")}
+--        	  ${if(len(gQtrMonth)=5,"/*","")}
+              ,sum(case when a.stat_date = ${gQtrMonth} then a.period_loc_amt else 0 end) as bq_amt
+              from b
+--------------------
+SELECT b.merge_region_code AS region_code, b.merge_region_name AS region_name, b.govern_region_area_union AS region_area_subtotal ${if(gRegionAreaLevel='合并营运区',",b.govern_region_area_union as region_area_code,b.govern_region_area_union as region_area_name",",b.region_area_code as region_area_code,b.region_area_name as region_area_name")} --        	  ${if(len(gQtrMonth)=5,"/*","")}
+	, sum(CASE
+		WHEN a.stat_date = ${gQtrMonth} THEN a.period_loc_amt
+		ELSE 0
+	END) AS bq_amt
+FROM b
+------------------------------------------------------------------------------------------------------------------------
+select 
+    ${if(gGyzy_hz=='G'," buyer_name as buyer_name, "," syr as buyer_name, ")}
+    substringUTF8(buyer_name,1,positionUTF8(buyer_name,'(')-1) as tg
+    from a
+--------------------
+SELECT ${if(gGyzy_hz=='G'," buyer_name as buyer_name, "," syr as buyer_name, ")}
+	, substringUTF8(buyer_name, 1, positionUTF8(buyer_name, '(') - 1) AS tg
+FROM a
+------------------------------------------------------------------------------------------------------------------------
+select c from b where c=1 ${if(len(a)=0,'and 1=1',"and a>0")} ${if(len(a)=0,'and 1=1',"and a>0")}
+--------------------
+SELECT c
+FROM b
+WHERE c = 1 ${if(len(a)=0,'and 1=1',"and a>0")} ${if(len(a)=0,'and 1=1',"and a>0")}
+------------------------------------------------------------------------------------------------------------------------
 --test
 select a from b
 --------------------

core/src/test/resources/bvt/parser/sqlserver/0.txt

@@ -12,6 +12,22 @@ select * from [test].[test]
 SELECT *
 FROM [test].[test]
 ------------------------------------------------------------------------------------------------------------------------
+CREATE VIEW [a].[b]
+AS SELECT
+*
+, [c] = DATEDIFF(MILLISECOND, request_time, GETDATE()) / 1000.0
+FROM
+tbl
+WHERE
+[state] = 'Queued';
+--------------------
+CREATE VIEW [a].[b]
+AS
+SELECT *
+	, [c] = DATEDIFF(MILLISECOND, request_time, GETDATE()) / 1000.0
+FROM tbl
+WHERE [state] = 'Queued';
+------------------------------------------------------------------------------------------------------------------------
 USE AdventureWorks2008R2
 --------------------
 USE AdventureWorks2008R2