Skip to content

Commit

Permalink
Merge pull request #116 from cbanek/tickets/DM-17872
Browse files Browse the repository at this point in the history 
[DM-17872] Parsing for math and coercing types
  • Loading branch information
@pdowler
pdowler authored Feb 22, 2024
2 parents 8bc296d + 3a8a15c commit 5039841
Show file tree
Hide file tree
Showing 4 changed files with 385 additions and 106 deletions.
262 changes: 183 additions & 79 deletions cadc-adql/src/main/java/ca/nrc/cadc/tap/parser/extractor/SelectListExpressionExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,18 @@
import ca.nrc.cadc.tap.schema.TapSchema;
import java.util.ArrayList;
import java.util.List;
import net.sf.jsqlparser.expression.BinaryExpression;
import net.sf.jsqlparser.expression.CaseExpression;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.Function;
import net.sf.jsqlparser.expression.DoubleValue;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.Parenthesis;
import net.sf.jsqlparser.expression.WhenClause;
import net.sf.jsqlparser.expression.operators.arithmetic.Addition;
import net.sf.jsqlparser.expression.operators.arithmetic.Division;
import net.sf.jsqlparser.expression.operators.arithmetic.Multiplication;
import net.sf.jsqlparser.expression.operators.arithmetic.Subtraction;
import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.statement.select.AllColumns;
Expand All @@ -103,15 +113,28 @@ public class SelectListExpressionExtractor extends ExpressionNavigator

protected TapSchema tapSchema;
protected List<TapSelectItem> selectList;
protected int columnIndex;
protected String alias;

/**
* @param tapSchema
*/
public SelectListExpressionExtractor(TapSchema tapSchema)
{
this(tapSchema, 1);
}

/**
* @param tapSchema
* @param columnIndex
*/
public SelectListExpressionExtractor(TapSchema tapSchema, int columnIndex)
{
super();
this.tapSchema = tapSchema;
this.selectList = new ArrayList<TapSelectItem>();
this.columnIndex = columnIndex;
this.alias = null;
}

/* (non-Javadoc)
Expand Down Expand Up @@ -139,40 +162,17 @@ public void visit(AllTableColumns allTableColumns)
public void visit(SelectExpressionItem selectExpressionItem)
{
log.debug("visit(selectExpressionItem)" + selectExpressionItem);

TapSelectItem paramDesc = null;
PlainSelect plainSelect = selectNavigator.getPlainSelect();
String alias = selectExpressionItem.getAlias();
if (alias != null && alias.isEmpty())
alias = null;


Expression expression = selectExpressionItem.getExpression();
if (expression instanceof Column)
{
Column column = (Column) expression;
ColumnDesc columnDesc = TapSchemaUtil.findColumnDesc(tapSchema, plainSelect, column);
log.debug("visit(column) " + column + "found: " + columnDesc);
if (alias != null)
paramDesc = new TapSelectItem(alias, columnDesc);
else
paramDesc = new TapSelectItem(column.getColumnName(), columnDesc);
}
else if (expression instanceof Function)
{
Function function = (Function) expression;
FunctionDesc functionDesc = getFunctionDesc(function, plainSelect);
log.debug("visit(function) " + function + " found: " + functionDesc);
if (alias != null)
paramDesc = new TapSelectItem(alias, functionDesc.getDatatype());
else
paramDesc = new TapSelectItem(function.getName(), functionDesc.getDatatype());
}
else if (expression instanceof SubSelect)
if (expression instanceof SubSelect)
{
SubSelect subSelect = (SubSelect) expression;
log.debug("visit(subSelect) " + subSelect);

SelectListExtractor sle = new SelectListExtractor(new SelectListExpressionExtractor(tapSchema),
SelectListExtractor sle = new SelectListExtractor(new SelectListExpressionExtractor(tapSchema, getColumnIndex()),
new ReferenceNavigator(),
new FromItemNavigator());
subSelect.getSelectBody().accept(sle);
Expand All @@ -187,14 +187,13 @@ else if (expression instanceof SubSelect)
}
else
{
TapDataType datatype = getDatatypeFromExpression(expression);
if (alias != null)
paramDesc = new TapSelectItem(alias, datatype);
else
paramDesc = new TapSelectItem(expression.toString(), datatype);
setAlias(selectExpressionItem.getAlias());
paramDesc = getItemFromExpression(expression, plainSelect);
}

log.debug("select item: " + paramDesc.getColumnName() + " " + paramDesc.getDatatype());
selectList.add(paramDesc);
setColumnIndex(selectList.size() + 1);
}

public List<TapSelectItem> getSelectList()
Expand All @@ -217,67 +216,172 @@ public void setTapSchema(TapSchema tapSchema)
this.tapSchema = tapSchema;
}

private FunctionDesc getFunctionDesc(Function function, PlainSelect plainSelect)
public int getColumnIndex()
{
return this.columnIndex;
}

public void setColumnIndex(int columnIndex)
{
this.columnIndex = columnIndex;
}

public String getAlias()
{
return this.alias;
}

public void setAlias(String alias)
{
this.alias = alias;
}

public String getColumnName(String prefix)
{
String alias = getAlias();

if(alias == null || alias.isEmpty())
return prefix + getColumnIndex();
else
return alias;
}

public String getGenericColumnName()
{
return getColumnName("col");
}

public boolean isIntegerNumericType(TapDataType dt)
{
return (TapDataType.BOOLEAN.equals(dt) ||
TapDataType.INTEGER.equals(dt) ||
TapDataType.SHORT.equals(dt) ||
TapDataType.LONG.equals(dt));
}

public boolean isNonIntegerNumericType(TapDataType dt)
{
return (TapDataType.DOUBLE.equals(dt) || TapDataType.FLOAT.equals(dt));
}

private TapSelectItem getItemFromFunction(Function function, PlainSelect plainSelect)
{
FunctionDesc functionDesc = TapSchemaUtil.findFunctionDesc(tapSchema, function);
log.debug("getFunctionDesc: " + function.getName() + " -> " + functionDesc);
String name = getColumnName(function.getName());

log.debug("getItemFromFunction: " + function.getName() + " -> " + functionDesc);
if (functionDesc == null)
throw new UnsupportedOperationException("invalid function: " + function.getName());

if ( TapDataType.FUNCTION_ARG.equals(functionDesc.getDatatype()) )
{
TapDataType datatype = null;
ExpressionList parameters = function.getParameters();
for (Object parameter : parameters.getExpressions())
// Some functions return the type of their arguments rather than a
// static type.
for (Object parameter : function.getParameters().getExpressions())
{
if (parameter instanceof Column)
{
ColumnDesc columnDesc = TapSchemaUtil.findColumnDesc(tapSchema, plainSelect, (Column) parameter);
if (columnDesc != null)
{
datatype = columnDesc.getDatatype();
//arg = columnDesc;
}
}
else if (parameter instanceof Function)
{
Function nestedFunction = (Function) parameter;
log.debug("vist(nested Function " + nestedFunction);
FunctionDesc nestedFunctionDesc = TapSchemaUtil.findFunctionDesc(tapSchema, (Function) parameter);
if ( TapDataType.FUNCTION_ARG.equals(nestedFunctionDesc.getDatatype()) )
{
FunctionDesc recursiveFunctionDesc = getFunctionDesc(nestedFunction, plainSelect);
if (recursiveFunctionDesc != null)
{
datatype = recursiveFunctionDesc.getDatatype();
//arg = recursiveFunctionDesc.arg;
}
}
else
{
datatype = nestedFunctionDesc.getDatatype();
//arg = nestedFunctionDesc.arg;
}
}
else
TapSelectItem item = getItemFromExpression((Expression) parameter, plainSelect);
return new TapSelectItem(name, item.getDatatype());
}
}

return new TapSelectItem(name, functionDesc.getDatatype());
}

private TapSelectItem getItemFromExpression(Expression expression, PlainSelect ps)
{
if (expression instanceof Column)
{
Column column = (Column)expression;
String name = column.getColumnName();
if (getAlias() != null && !getAlias().isEmpty())
name = getAlias();

ColumnDesc columnDesc = TapSchemaUtil.findColumnDesc(tapSchema, ps, column);
return new TapSelectItem(name, columnDesc);
}
else if (expression instanceof Function)
{
return getItemFromFunction((Function)expression, ps);
}
else if (expression instanceof Parenthesis)
{
Parenthesis parenthesis = (Parenthesis)expression;
return getItemFromExpression(parenthesis.getExpression(), ps);
}
else if (expression instanceof CaseExpression)
{
CaseExpression ce = (CaseExpression)expression;
List<WhenClause> clauses = ce.getWhenClauses();
List<Expression> expressions = new ArrayList<Expression>();

for (WhenClause wc : clauses)
expressions.add(wc.getThenExpression());

if (ce.getElseExpression() != null)
expressions.add(ce.getElseExpression());

TapDataType datatype = TapDataType.LONG;

for (Expression exp : expressions)
{
TapSelectItem i = getItemFromExpression(exp, ps);
TapDataType dt = i.getDatatype();

if (!isIntegerNumericType(i.getDatatype()) &&
!isNonIntegerNumericType(i.getDatatype()))
{
datatype = getDatatypeFromExpression((Expression) parameter);
// If any of the then expressions return a non-numeric, we
// have to assume the worst and always return a string.
return new TapSelectItem(getGenericColumnName(), TapDataType.STRING);
}

if (datatype != null)
else if(isNonIntegerNumericType(i.getDatatype()))
{
return new FunctionDesc(functionDesc.getName(), datatype);
// If any of the then expressions return a double,
// the datatype is double
datatype = TapDataType.DOUBLE;
}
}

return new TapSelectItem(getGenericColumnName(), datatype);
}
log.debug("getFunctionDesc: " + function.getName() + " -> " + functionDesc);
return functionDesc;
}
else if (expression instanceof DoubleValue)
{
return new TapSelectItem(getGenericColumnName(), TapDataType.DOUBLE);
}
else if (expression instanceof LongValue)
{
return new TapSelectItem(getGenericColumnName(), TapDataType.LONG);
}
else if (expression instanceof Addition ||
expression instanceof Subtraction ||
expression instanceof Multiplication)
{
BinaryExpression be = (BinaryExpression)expression;
TapSelectItem left = getItemFromExpression(be.getLeftExpression(), ps);
TapSelectItem right = getItemFromExpression(be.getRightExpression(), ps);
TapDataType leftType = left.getDatatype();
TapDataType rightType = right.getDatatype();

private TapDataType getDatatypeFromExpression(Expression expression)
{
// TODO: could check constant types instead iof lazy string
return new TapDataType("char", "*", null);
log.debug("leftType: " + leftType + " rightType: " + rightType);

TapDataType datatype;

if (isIntegerNumericType(leftType) && isIntegerNumericType(rightType))
datatype = TapDataType.LONG;
else
datatype = TapDataType.DOUBLE;

return new TapSelectItem(getGenericColumnName(), datatype);
}
else if (expression instanceof Division)
{
return new TapSelectItem(getGenericColumnName(), TapDataType.DOUBLE);
}
else
{
// Default to a string representation with a generic name
// that can carry anything.
return new TapSelectItem(getGenericColumnName(), TapDataType.STRING);
}
}

}
4 changes: 2 additions & 2 deletions cadc-adql/src/test/java/ca/nrc/cadc/tap/AdqlQueryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,15 +199,15 @@ public void testSubSelectInSelect()
selectList = doit();
assertTrue(selectList.size() == 2);
tsi = selectList.get(1);
assertEquals("count", tsi.getName().toLowerCase());
assertEquals("count2", tsi.getName().toLowerCase());
assertEquals("long", tsi.getDatatype().getDatatype());

_query = "select schema_name, (select count(*) from tap_schema.alldatatypes) from tap_schema.tables";
_expected = "select schema_name, (select count(*) from tap_schema.alldatatypes) from tap_schema.tables";
selectList = doit();
assertTrue(selectList.size() == 2);
tsi = selectList.get(1);
assertEquals("count", tsi.getName().toLowerCase());
assertEquals("count2", tsi.getName().toLowerCase());
assertEquals("long", tsi.getDatatype().getDatatype());
}

Expand Down
6 changes: 3 additions & 3 deletions cadc-adql/src/test/java/ca/nrc/cadc/tap/parser/TestUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,9 +131,9 @@ public static TapSchema mockTapSchema()
td = new TableDesc(sn, tn);
sd.getTableDescs().add(td);
td.getColumnDescs().add( createColumnDesc(tn, "t_integer", TapDataType.INTEGER, "int column", null, null, null));
td.getColumnDescs().add( new ColumnDesc(tn, "t_long", TapDataType.INTEGER));
td.getColumnDescs().add( new ColumnDesc(tn, "t_float", TapDataType.INTEGER));
td.getColumnDescs().add( new ColumnDesc(tn, "t_double", TapDataType.INTEGER));
td.getColumnDescs().add( new ColumnDesc(tn, "t_long", TapDataType.LONG));
td.getColumnDescs().add( new ColumnDesc(tn, "t_float", TapDataType.FLOAT));
td.getColumnDescs().add( new ColumnDesc(tn, "t_double", TapDataType.DOUBLE));
td.getColumnDescs().add( new ColumnDesc(tn, "t_char", new TapDataType("char", "8", null)));
td.getColumnDescs().add( createColumnDesc(tn, "t_varchar", new TapDataType("char", "8*", null), "varchar column", null, null, null));
td.getColumnDescs().add( new ColumnDesc(tn, "t_string", new TapDataType("char", "8*", null)));
Expand Down
Loading

0 comments on commit 5039841

Please sign in to comment.