feat: ON OVERFLOW support for the ListAgg function (Oracle, DB2)

- fixes #2003

Signed-off-by: Andreas Reichel <[email protected]>

src/main/java/net/sf/jsqlparser/expression/AnalyticExpression.java

@@ -40,6 +40,8 @@ public class AnalyticExpression extends ASTNodeAccessImpl implements Expression
     private boolean ignoreNullsOutside = false; // IGNORE NULLS outside function parameters
     private Expression filterExpression = null;
     private List<OrderByElement> funcOrderBy = null;
+    private String onOverflowTruncate = null;
+
     private String windowName = null; // refers to an external window definition (paritionBy,
     // orderBy, windowElement)
     private WindowDefinition windowDef = new WindowDefinition();
@@ -78,6 +80,7 @@ public AnalyticExpression(Function function) {
         this.ignoreNullsOutside = function.isIgnoreNullsOutside();
         this.nullHandling = function.getNullHandling();
         this.funcOrderBy = function.getOrderByElements();
+        this.onOverflowTruncate = function.getOnOverflowTruncate();
         this.limit = function.getLimit();
         this.keep = function.getKeep();
     }
@@ -96,6 +99,15 @@ public void setOrderByElements(List<OrderByElement> orderByElements) {
         windowDef.orderBy.setOrderByElements(orderByElements);
     }
 
+    public String getOnOverflowTruncate() {
+        return onOverflowTruncate;
+    }
+
+    public AnalyticExpression setOnOverflowTruncate(String onOverflowTruncate) {
+        this.onOverflowTruncate = onOverflowTruncate;
+        return this;
+    }
+
     public KeepExpression getKeep() {
         return keep;
     }
@@ -294,6 +306,10 @@ public String toString() {
             b.append(funcOrderBy.stream().map(OrderByElement::toString).collect(joining(", ")));
         }
 
+        if (onOverflowTruncate != null) {
+            b.append(" ON OVERFLOW ").append(onOverflowTruncate);
+        }
+
         if (limit != null) {
             b.append(limit);
         }

src/main/java/net/sf/jsqlparser/expression/Function.java

@@ -38,6 +38,7 @@ public class Function extends ASTNodeAccessImpl implements Expression {
     private boolean ignoreNullsOutside = false; // IGNORE NULLS outside function parameters
     private Limit limit = null;
     private KeepExpression keep = null;
+    private String onOverflowTruncate = null;
 
     public Function() {}
 
@@ -302,6 +303,11 @@ public String toString() {
                 if (limit != null) {
                     b.append(limit);
                 }
+
+                if (onOverflowTruncate != null) {
+                    b.append(" ON OVERFLOW ").append(onOverflowTruncate);
+                }
+
                 b.append(")");
                 params = b.toString();
             } else {
@@ -399,6 +405,15 @@ public void setOrderByElements(List<OrderByElement> orderByElements) {
         this.orderByElements = orderByElements;
     }
 
+    public String getOnOverflowTruncate() {
+        return onOverflowTruncate;
+    }
+
+    public Function setOnOverflowTruncate(String onOverflowTruncate) {
+        this.onOverflowTruncate = onOverflowTruncate;
+        return this;
+    }
+
     public <E extends Expression> E getAttribute(Class<E> type) {
         return type.cast(getAttribute());
     }

src/main/java/net/sf/jsqlparser/util/deparser/ExpressionDeParser.java

@@ -857,6 +857,11 @@ public <S> StringBuilder visit(Function function, S context) {
                     orderByDeParser.deParseElement(orderByElement);
                 }
             }
+
+            if (function.getOnOverflowTruncate() != null) {
+                buffer.append(" ON OVERFLOW ").append(function.getOnOverflowTruncate());
+            }
+
             if (function.getLimit() != null) {
                 new LimitDeparser(this, buffer).deParse(function.getLimit());
             }
@@ -1127,6 +1132,10 @@ public <S> StringBuilder visit(AnalyticExpression analyticExpression, S context)
                     .collect(joining(", ")));
         }
 
+        if (analyticExpression.getOnOverflowTruncate() != null) {
+            buffer.append(" ON OVERFLOW ").append(analyticExpression.getOnOverflowTruncate());
+        }
+
         if (analyticExpression.getLimit() != null) {
             new LimitDeparser(this, buffer).deParse(analyticExpression.getLimit());
         }

src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt

@@ -212,6 +212,7 @@ TOKEN: /* SQL Keywords. prefixed with K_ to avoid name clashes */
 |   <K_CONSTRAINTS:"CONSTRAINTS">
 |   <K_CONVERT:"CONVERT">
 |   <K_COSTS: "COSTS">
+|   <K_COUNT: "COUNT">
 |   <K_CREATE:"CREATE">
 |   <K_CROSS:"CROSS">
 |   <K_CURRENT: "CURRENT">
@@ -241,12 +242,13 @@ TOKEN: /* SQL Keywords. prefixed with K_ to avoid name clashes */
 |   <K_DUPLICATE: "DUPLICATE">
 |   <K_ELEMENTS: "ELEMENTS">
 |   <K_ELSE:"ELSE">
-|   <K_EMIT : "EMIT">
-|   <K_ENABLE : "ENABLE">
-|   <K_END:"END">
-|   <K_ESCAPE:"ESCAPE">
-|   <K_EXCEPT:"EXCEPT">
-|   <K_EXCLUDE : "EXCLUDE">
+|   <K_EMIT: "EMIT">
+|   <K_ENABLE: "ENABLE">
+|   <K_END: "END">
+|   <K_ERROR: "ERROR">
+|   <K_ESCAPE: "ESCAPE">
+|   <K_EXCEPT: "EXCEPT">
+|   <K_EXCLUDE: "EXCLUDE">
 |   <K_EXCLUDES: "EXCLUDES"> /* Salesforce SOQL */
 |   <K_EXEC: "EXEC">
 |   <K_EXECUTE: "EXECUTE">
@@ -370,6 +372,7 @@ TOKEN: /* SQL Keywords. prefixed with K_ to avoid name clashes */
 |   <K_OUTER:"OUTER">
 |   <K_OUTPUT:"OUTPUT">
 |   <K_OVER:"OVER">
+|   <K_OVERFLOW:"OVERFLOW">
 |   <K_OVERLAPS:"OVERLAPS">
 |   <K_OPTIMIZE: "OPTIMIZE" >
 |   <K_PARALLEL:"PARALLEL">
@@ -2015,7 +2018,7 @@ String RelObjectNameWithoutValue() :
 {    Token tk = null; }
 {
     ( tk=<DATA_TYPE> | tk=<S_IDENTIFIER> | tk=<S_QUOTED_IDENTIFIER> |  tk=<K_DATE_LITERAL> | tk=<K_DATETIMELITERAL> | tk=<K_STRING_FUNCTION_NAME> | tk=<K_ISOLATION> | tk=<K_TIME_KEY_EXPR> | tk=<K_TEXT_LITERAL> 
-       | tk="ACTION" | tk="ACTIVE" | tk="ADD" | tk="ADVANCE" | tk="ADVISE" | tk="AGAINST" | tk="ALGORITHM" | tk="ALTER" | tk="ANALYZE" | tk="APPLY" | tk="APPROXIMATE" | tk="ARCHIVE" | tk="ARRAY" | tk="ASC" | tk="AT" | tk="AUTHORIZATION" | tk="AUTO" | tk="BASE64" | tk="BEGIN" | tk="BERNOULLI" | tk="BINARY" | tk="BIT" | tk="BLOCK" | tk="BROWSE" | tk="BUFFERS" | tk="BY" | tk="BYTE" | tk="BYTES" | tk="CACHE" | tk="CALL" | tk="CASCADE" | tk="CASE" | tk="CAST" | tk="CHANGE" | tk="CHANGES" | tk="CHAR" | tk="CHARACTER" | tk="CHECKPOINT" | tk="CLOSE" | tk="COLLATE" | tk="COLUMN" | tk="COLUMNS" | tk="COMMENT" | tk="COMMIT" | tk="CONCURRENTLY" | tk="CONFLICT" | tk="CONSTRAINTS" | tk="CONVERT" | tk="COSTS" | tk="CS" | tk="CYCLE" | tk="DATA" | tk="DATABASE" | tk="DATETIME" | tk="DBA_RECYCLEBIN" | tk="DDL" | tk="DECLARE" | tk="DEFERRABLE" | tk="DELAYED" | tk="DELETE" | tk="DESC" | tk="DESCRIBE" | tk="DISABLE" | tk="DISCONNECT" | tk="DIV" | tk="DML" | tk="DO" | tk="DOMAIN" | tk="DROP" | tk="DUMP" | tk="DUPLICATE" | tk="ELEMENTS" | tk="EMIT" | tk="ENABLE" | tk="END" | tk="ESCAPE" | tk="EXCLUDE" | tk="EXEC" | tk="EXECUTE" | tk="EXPLAIN" | tk="EXPLICIT" | tk="EXTENDED" | tk="EXTRACT" | tk="FALSE" | tk="FILTER" | tk="FIRST" | tk="FLUSH" | tk="FN" | tk="FOLLOWING" | tk="FORMAT" | tk="FULLTEXT" | tk="FUNCTION" | tk="GRANT" | tk="GROUP_CONCAT" | tk="GUARD" | tk="HASH" | tk="HIGH_PRIORITY" | tk="HISTORY" | tk="HOPPING" | tk="INCLUDE" | tk="INCLUDE_NULL_VALUES" | tk="INCREMENT" | tk="INDEX" | tk="INSERT" | tk="INTERLEAVE" | tk="INTERPRET" | tk="INVALIDATE" | tk="ISNULL" | tk="JSON" | tk="JSON_ARRAY" | tk="JSON_ARRAYAGG" | tk="JSON_OBJECT" | tk="JSON_OBJECTAGG" | tk="KEEP" | tk="KEY" | tk="KEYS" | tk="LAST" | tk="LEADING" | tk="LINK" | tk="LOCAL" | tk="LOCKED" | tk="LOG" | tk="LONGTEXT" | tk="LOOP" | tk="LOW_PRIORITY" | tk="MATCH" | tk="MATCHED" | tk="MATERIALIZED" | tk="MAX" | tk="MAXVALUE" | tk="MEDIUMTEXT" | tk="MEMBER" | tk="MERGE" | tk="MIN" | tk="MINVALUE" | tk="MODIFY" | tk="MOVEMENT" | tk="NEXT" | tk="NO" | tk="NOCACHE" | tk="NOKEEP" | tk="NOLOCK" | tk="NOMAXVALUE" | tk="NOMINVALUE" | tk="NOORDER" | tk="NOTHING" | tk="NOTNULL" | tk="NOVALIDATE" | tk="NOWAIT" | tk="NULLS" | tk="OF" | tk="OFF" | tk="OPEN" | tk="OVER" | tk="OVERLAPS" | tk="PARALLEL" | tk="PARENT" | tk="PARTITION" | tk="PATH" | tk="PERCENT" | tk="PLACING" | tk="PLAN" | tk="PRECEDING" | tk="PRIMARY" | tk="PRIOR" | tk="PURGE" | tk="QUERY" | tk="QUICK" | tk="QUIESCE" | tk="RANGE" | tk="RAW" | tk="READ" | tk="RECURSIVE" | tk="RECYCLEBIN" | tk="REFERENCES" | tk="REFRESH" | tk="REGEXP" | tk="REGEXP_LIKE" | tk="REGISTER" | tk="REMOTE" | tk="RENAME" | tk="REPEATABLE" | tk="REPLACE" | tk="RESET" | tk="RESPECT" | tk="RESTART" | tk="RESTRICT" | tk="RESTRICTED" | tk="RESUMABLE" | tk="RESUME" | tk="RETURN" | tk="RLIKE" | tk="ROLLBACK" | tk="ROLLUP" | tk="ROOT" | tk="ROW" | tk="ROWS" | tk="RR" | tk="RS" | tk="SAFE_CAST" | tk="SAVEPOINT" | tk="SCHEMA" | tk="SECURE" | tk="SEED" | tk="SEPARATOR" | tk="SEQUENCE" | tk="SESSION" | tk="SETS" | tk="SHARE" | tk="SHOW" | tk="SHUTDOWN" | tk="SIBLINGS" | tk="SIGNED" | tk="SIMILAR" | tk="SIZE" | tk="SKIP" | tk="STORED" | tk="STRING" | tk="STRUCT" | tk="SUMMARIZE" | tk="SUSPEND" | tk="SWITCH" | tk="SYNONYM" | tk="SYSTEM" | tk="TABLE" | tk="TABLESPACE" | tk="TEMP" | tk="TEMPORARY" | tk="TEXT" | tk="THEN" | tk="TIMEOUT" | tk="TIMESTAMPTZ" | tk="TIMEZONE" | tk="TINYTEXT" | tk="TO" | tk="TRIGGER" | tk="TRUE" | tk="TRUNCATE" | tk="TRY_CAST" | tk="TUMBLING" | tk="TYPE" | tk="UNLOGGED" | tk="UNQIESCE" | tk="UNSIGNED" | tk="UPDATE" | tk="UPSERT" | tk="UR" | tk="USER" | tk="VALIDATE" | tk="VERBOSE" | tk="VIEW" | tk="VOLATILE" | tk="WAIT" | tk="WITHIN" | tk="WITHOUT" | tk="WITHOUT_ARRAY_WRAPPER" | tk="WORK" | tk="XML" | tk="XMLAGG" | tk="XMLDATA" | tk="XMLSCHEMA" | tk="XMLTEXT" | tk="XSINIL" | tk="YAML" | tk="YES" | tk="ZONE" )
+       | tk="ACTION" | tk="ACTIVE" | tk="ADD" | tk="ADVANCE" | tk="ADVISE" | tk="AGAINST" | tk="ALGORITHM" | tk="ALTER" | tk="ANALYZE" | tk="APPLY" | tk="APPROXIMATE" | tk="ARCHIVE" | tk="ARRAY" | tk="ASC" | tk="AT" | tk="AUTHORIZATION" | tk="AUTO" | tk="BASE64" | tk="BEGIN" | tk="BERNOULLI" | tk="BINARY" | tk="BIT" | tk="BLOCK" | tk="BROWSE" | tk="BUFFERS" | tk="BY" | tk="BYTE" | tk="BYTES" | tk="CACHE" | tk="CALL" | tk="CASCADE" | tk="CASE" | tk="CAST" | tk="CHANGE" | tk="CHANGES" | tk="CHAR" | tk="CHARACTER" | tk="CHECKPOINT" | tk="CLOSE" | tk="COLLATE" | tk="COLUMN" | tk="COLUMNS" | tk="COMMENT" | tk="COMMIT" | tk="CONCURRENTLY" | tk="CONFLICT" | tk="CONSTRAINTS" | tk="CONVERT" | tk="COSTS" | tk="COUNT" | tk="CS" | tk="CYCLE" | tk="DATA" | tk="DATABASE" | tk="DATETIME" | tk="DBA_RECYCLEBIN" | tk="DDL" | tk="DECLARE" | tk="DEFERRABLE" | tk="DELAYED" | tk="DELETE" | tk="DESC" | tk="DESCRIBE" | tk="DISABLE" | tk="DISCONNECT" | tk="DIV" | tk="DML" | tk="DO" | tk="DOMAIN" | tk="DROP" | tk="DUMP" | tk="DUPLICATE" | tk="ELEMENTS" | tk="EMIT" | tk="ENABLE" | tk="END" | tk="ERROR" | tk="ESCAPE" | tk="EXCLUDE" | tk="EXEC" | tk="EXECUTE" | tk="EXPLAIN" | tk="EXPLICIT" | tk="EXTENDED" | tk="EXTRACT" | tk="FALSE" | tk="FILTER" | tk="FIRST" | tk="FLUSH" | tk="FN" | tk="FOLLOWING" | tk="FORMAT" | tk="FULLTEXT" | tk="FUNCTION" | tk="GRANT" | tk="GROUP_CONCAT" | tk="GUARD" | tk="HASH" | tk="HIGH_PRIORITY" | tk="HISTORY" | tk="HOPPING" | tk="INCLUDE" | tk="INCLUDE_NULL_VALUES" | tk="INCREMENT" | tk="INDEX" | tk="INSERT" | tk="INTERLEAVE" | tk="INTERPRET" | tk="INVALIDATE" | tk="ISNULL" | tk="JSON" | tk="JSON_ARRAY" | tk="JSON_ARRAYAGG" | tk="JSON_OBJECT" | tk="JSON_OBJECTAGG" | tk="KEEP" | tk="KEY" | tk="KEYS" | tk="LAST" | tk="LEADING" | tk="LINK" | tk="LOCAL" | tk="LOCKED" | tk="LOG" | tk="LONGTEXT" | tk="LOOP" | tk="LOW_PRIORITY" | tk="MATCH" | tk="MATCHED" | tk="MATERIALIZED" | tk="MAX" | tk="MAXVALUE" | tk="MEDIUMTEXT" | tk="MEMBER" | tk="MERGE" | tk="MIN" | tk="MINVALUE" | tk="MODIFY" | tk="MOVEMENT" | tk="NEXT" | tk="NO" | tk="NOCACHE" | tk="NOKEEP" | tk="NOLOCK" | tk="NOMAXVALUE" | tk="NOMINVALUE" | tk="NOORDER" | tk="NOTHING" | tk="NOTNULL" | tk="NOVALIDATE" | tk="NOWAIT" | tk="NULLS" | tk="OF" | tk="OFF" | tk="OPEN" | tk="OVER" | tk="OVERFLOW" | tk="OVERLAPS" | tk="PARALLEL" | tk="PARENT" | tk="PARTITION" | tk="PATH" | tk="PERCENT" | tk="PLACING" | tk="PLAN" | tk="PRECEDING" | tk="PRIMARY" | tk="PRIOR" | tk="PURGE" | tk="QUERY" | tk="QUICK" | tk="QUIESCE" | tk="RANGE" | tk="RAW" | tk="READ" | tk="RECURSIVE" | tk="RECYCLEBIN" | tk="REFERENCES" | tk="REFRESH" | tk="REGEXP" | tk="REGEXP_LIKE" | tk="REGISTER" | tk="REMOTE" | tk="RENAME" | tk="REPEATABLE" | tk="REPLACE" | tk="RESET" | tk="RESPECT" | tk="RESTART" | tk="RESTRICT" | tk="RESTRICTED" | tk="RESUMABLE" | tk="RESUME" | tk="RETURN" | tk="RLIKE" | tk="ROLLBACK" | tk="ROLLUP" | tk="ROOT" | tk="ROW" | tk="ROWS" | tk="RR" | tk="RS" | tk="SAFE_CAST" | tk="SAVEPOINT" | tk="SCHEMA" | tk="SECURE" | tk="SEED" | tk="SEPARATOR" | tk="SEQUENCE" | tk="SESSION" | tk="SETS" | tk="SHARE" | tk="SHOW" | tk="SHUTDOWN" | tk="SIBLINGS" | tk="SIGNED" | tk="SIMILAR" | tk="SIZE" | tk="SKIP" | tk="STORED" | tk="STRING" | tk="STRUCT" | tk="SUMMARIZE" | tk="SUSPEND" | tk="SWITCH" | tk="SYNONYM" | tk="SYSTEM" | tk="TABLE" | tk="TABLESPACE" | tk="TEMP" | tk="TEMPORARY" | tk="TEXT" | tk="THEN" | tk="TIMEOUT" | tk="TIMESTAMPTZ" | tk="TIMEZONE" | tk="TINYTEXT" | tk="TO" | tk="TRIGGER" | tk="TRUE" | tk="TRUNCATE" | tk="TRY_CAST" | tk="TUMBLING" | tk="TYPE" | tk="UNLOGGED" | tk="UNQIESCE" | tk="UNSIGNED" | tk="UPDATE" | tk="UPSERT" | tk="UR" | tk="USER" | tk="VALIDATE" | tk="VERBOSE" | tk="VIEW" | tk="VOLATILE" | tk="WAIT" | tk="WITHIN" | tk="WITHOUT" | tk="WITHOUT_ARRAY_WRAPPER" | tk="WORK" | tk="XML" | tk="XMLAGG" | tk="XMLDATA" | tk="XMLSCHEMA" | tk="XMLTEXT" | tk="XSINIL" | tk="YAML" | tk="YES" | tk="ZONE" )
     { return tk.image; }
 }
 
@@ -5823,6 +5826,8 @@ Function InternalFunction(boolean escaped):
     Expression attributeExpression = null;
     Column attributeColumn = null;
     List<OrderByElement> orderByList;
+    String onOverflowTruncate = null;
+    Token overflowToken = null;
     Limit limit;
 }
 {
@@ -5843,7 +5848,18 @@ Function InternalFunction(boolean escaped):
             |
             LOOKAHEAD( AllTableColumns() ) expr=AllTableColumns() { expressionList = new ExpressionList(expr); }
             |
-            LOOKAHEAD(3) expressionList=ExpressionList() [ orderByList = OrderByElements() { retval.setOrderByElements(orderByList); } ]
+            LOOKAHEAD(3) expressionList=ExpressionList()
+            [ orderByList = OrderByElements() { retval.setOrderByElements(orderByList); } ]
+
+            // https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/LISTAGG.html
+            [
+                <K_ON> <K_OVERFLOW> ( overflowToken=<K_TRUNCATE> | overflowToken=<K_ERROR> ) { onOverflowTruncate=overflowToken.image; }
+                [
+                    overflowToken = <S_CHAR_LITERAL> { onOverflowTruncate+= " " + overflowToken.image; }
+                    [ ( overflowToken=<K_WITH> |  overflowToken=<K_WITHOUT> ) <K_COUNT> { onOverflowTruncate+=" " + overflowToken.image + " COUNT"; }]
+                ]
+            ] { retval.setOnOverflowTruncate(onOverflowTruncate); }
+
             |
             LOOKAHEAD({ !getAsBoolean(Feature.allowUnparenthesizedSubSelects) }) expr = Select() { expressionList = new ExpressionList(expr); }
         )

src/test/java/net/sf/jsqlparser/expression/FunctionTest.java

@@ -13,6 +13,8 @@
 import net.sf.jsqlparser.test.TestUtils;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 
 class FunctionTest {
     @Test
@@ -90,4 +92,20 @@ void testSimpleFunctionIssue2059() throws JSQLParserException {
             parser.withAllowComplexParsing(false);
         });
     }
+
+    @ParameterizedTest
+    @ValueSource(strings = {
+            "select LISTAGG(field, ',' on overflow truncate '...') from dual",
+            "select LISTAGG(field, ',' on overflow truncate '...' with count) from dual",
+            "select LISTAGG(field, ',' on overflow truncate '...' without count) from dual",
+            "select LISTAGG(field, ',' on overflow error) from dual", "SELECT department, \n" +
+                    "       LISTAGG(name, ', ' ON OVERFLOW TRUNCATE '... (truncated)' WITH COUNT) WITHIN GROUP (ORDER BY name)\n"
+                    +
+                    "       AS employee_names\n" +
+                    "FROM employees\n" +
+                    "GROUP BY department;"
+    })
+    void testListAggOnOverflow(String sqlStr) throws Exception {
+        TestUtils.assertSqlCanBeParsedAndDeparsed(sqlStr, true);
+    }
 }