Skip to content

Commit

Permalink
[DM-17872] Parsing for math and coercing types
Browse files Browse the repository at this point in the history
  • Loading branch information
@cbanek
cbanek committed Dec 21, 2023
1 parent 246b05f commit 50a861a
Show file tree
Hide file tree
Showing 3 changed files with 114 additions and 94 deletions.
166 changes: 85 additions & 81 deletions cadc-adql/src/main/java/ca/nrc/cadc/tap/parser/extractor/SelectListExpressionExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,16 @@
import ca.nrc.cadc.tap.schema.TapSchema;
import java.util.ArrayList;
import java.util.List;
import net.sf.jsqlparser.expression.BinaryExpression;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.Function;
import net.sf.jsqlparser.expression.DoubleValue;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.Parenthesis;
import net.sf.jsqlparser.expression.operators.arithmetic.Addition;
import net.sf.jsqlparser.expression.operators.arithmetic.Division;
import net.sf.jsqlparser.expression.operators.arithmetic.Multiplication;
import net.sf.jsqlparser.expression.operators.arithmetic.Subtraction;
import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.statement.select.AllColumns;
Expand All @@ -103,15 +111,26 @@ public class SelectListExpressionExtractor extends ExpressionNavigator

protected TapSchema tapSchema;
protected List<TapSelectItem> selectList;
protected int columnIndex;

/**
* @param tapSchema
*/
public SelectListExpressionExtractor(TapSchema tapSchema)
{
this(tapSchema, 1);
}

/**
* @param tapSchema
* @param columnIndex
*/
public SelectListExpressionExtractor(TapSchema tapSchema, int columnIndex)
{
super();
this.tapSchema = tapSchema;
this.selectList = new ArrayList<TapSelectItem>();
this.columnIndex = columnIndex;
}

/* (non-Javadoc)
Expand Down Expand Up @@ -139,40 +158,17 @@ public void visit(AllTableColumns allTableColumns)
public void visit(SelectExpressionItem selectExpressionItem)
{
log.debug("visit(selectExpressionItem)" + selectExpressionItem);

TapSelectItem paramDesc = null;
PlainSelect plainSelect = selectNavigator.getPlainSelect();
String alias = selectExpressionItem.getAlias();
if (alias != null && alias.isEmpty())
alias = null;


Expression expression = selectExpressionItem.getExpression();
if (expression instanceof Column)
{
Column column = (Column) expression;
ColumnDesc columnDesc = TapSchemaUtil.findColumnDesc(tapSchema, plainSelect, column);
log.debug("visit(column) " + column + "found: " + columnDesc);
if (alias != null)
paramDesc = new TapSelectItem(alias, columnDesc);
else
paramDesc = new TapSelectItem(column.getColumnName(), columnDesc);
}
else if (expression instanceof Function)
{
Function function = (Function) expression;
FunctionDesc functionDesc = getFunctionDesc(function, plainSelect);
log.debug("visit(function) " + function + " found: " + functionDesc);
if (alias != null)
paramDesc = new TapSelectItem(alias, functionDesc.getDatatype());
else
paramDesc = new TapSelectItem(function.getName(), functionDesc.getDatatype());
}
else if (expression instanceof SubSelect)
if (expression instanceof SubSelect)
{
SubSelect subSelect = (SubSelect) expression;
log.debug("visit(subSelect) " + subSelect);

SelectListExtractor sle = new SelectListExtractor(new SelectListExpressionExtractor(tapSchema),
SelectListExtractor sle = new SelectListExtractor(new SelectListExpressionExtractor(tapSchema, columnIndex),
new ReferenceNavigator(),
new FromItemNavigator());
subSelect.getSelectBody().accept(sle);
Expand All @@ -187,14 +183,16 @@ else if (expression instanceof SubSelect)
}
else
{
TapDataType datatype = getDatatypeFromExpression(expression);
if (alias != null)
paramDesc = new TapSelectItem(alias, datatype);
else
paramDesc = new TapSelectItem(expression.toString(), datatype);
String alias = selectExpressionItem.getAlias();
if (alias != null && alias.isEmpty())
alias = null;

paramDesc = getItemFromExpression(expression, plainSelect, alias);
}

log.debug("select item: " + paramDesc.getColumnName() + " " + paramDesc.getDatatype());
selectList.add(paramDesc);
columnIndex = selectList.size() + 1;
}

public List<TapSelectItem> getSelectList()
Expand All @@ -217,67 +215,73 @@ public void setTapSchema(TapSchema tapSchema)
this.tapSchema = tapSchema;
}

private FunctionDesc getFunctionDesc(Function function, PlainSelect plainSelect)
public int getColumnIndex()
{
return this.columnIndex;
}

public void setColumnIndex(int columnIndex)
{
this.columnIndex = columnIndex;
}

private TapSelectItem getItemFromFunction(Function function, PlainSelect plainSelect, String alias)
{
FunctionDesc functionDesc = TapSchemaUtil.findFunctionDesc(tapSchema, function);
log.debug("getFunctionDesc: " + function.getName() + " -> " + functionDesc);
String name = function.getName() + columnIndex;
if (alias != null)
name = alias;

log.debug("getItemFromFunction: " + function.getName() + " -> " + functionDesc);
if (functionDesc == null)
throw new UnsupportedOperationException("invalid function: " + function.getName());

if ( TapDataType.FUNCTION_ARG.equals(functionDesc.getDatatype()) )
{
TapDataType datatype = null;
ExpressionList parameters = function.getParameters();
for (Object parameter : parameters.getExpressions())
// Some functions return the type of their arguments rather than a
// static type.
for (Object parameter : function.getParameters().getExpressions())
{
if (parameter instanceof Column)
{
ColumnDesc columnDesc = TapSchemaUtil.findColumnDesc(tapSchema, plainSelect, (Column) parameter);
if (columnDesc != null)
{
datatype = columnDesc.getDatatype();
//arg = columnDesc;
}
}
else if (parameter instanceof Function)
{
Function nestedFunction = (Function) parameter;
log.debug("vist(nested Function " + nestedFunction);
FunctionDesc nestedFunctionDesc = TapSchemaUtil.findFunctionDesc(tapSchema, (Function) parameter);
if ( TapDataType.FUNCTION_ARG.equals(nestedFunctionDesc.getDatatype()) )
{
FunctionDesc recursiveFunctionDesc = getFunctionDesc(nestedFunction, plainSelect);
if (recursiveFunctionDesc != null)
{
datatype = recursiveFunctionDesc.getDatatype();
//arg = recursiveFunctionDesc.arg;
}
}
else
{
datatype = nestedFunctionDesc.getDatatype();
//arg = nestedFunctionDesc.arg;
}
}
else
{
datatype = getDatatypeFromExpression((Expression) parameter);
}

if (datatype != null)
{
return new FunctionDesc(functionDesc.getName(), datatype);
}
TapSelectItem item = getItemFromExpression((Expression) parameter, plainSelect, alias);
return new TapSelectItem(name, item.getDatatype());
}
}
log.debug("getFunctionDesc: " + function.getName() + " -> " + functionDesc);
return functionDesc;

return new TapSelectItem(name, functionDesc.getDatatype());
}

private TapDataType getDatatypeFromExpression(Expression expression)
private TapSelectItem getItemFromExpression(Expression expression, PlainSelect ps, String alias)
{
// TODO: could check constant types instead iof lazy string
return new TapDataType("char", "*", null);
if (expression instanceof Column)
{
Column column = (Column)expression;
String name = column.getColumnName();
if (alias != null)
name = alias;

ColumnDesc columnDesc = TapSchemaUtil.findColumnDesc(tapSchema, ps, column);
return new TapSelectItem(name, columnDesc);
}
else if (expression instanceof Function)
{
return getItemFromFunction((Function)expression, ps, alias);
}
else if (expression instanceof Parenthesis)
{
Parenthesis parenthesis = (Parenthesis)expression;
return getItemFromExpression(parenthesis.getExpression(), ps, alias);
}

// If all else fails, defaults to long with the name being
// a generic column name.
TapDataType datatype = TapDataType.LONG;
String name = "col";

name = name + columnIndex;

if (alias != null)
name = alias;

return new TapSelectItem(name, datatype);
}

}
4 changes: 2 additions & 2 deletions cadc-adql/src/test/java/ca/nrc/cadc/tap/AdqlQueryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,15 +199,15 @@ public void testSubSelectInSelect()
selectList = doit();
assertTrue(selectList.size() == 2);
tsi = selectList.get(1);
assertEquals("count", tsi.getName().toLowerCase());
assertEquals("count2", tsi.getName().toLowerCase());
assertEquals("long", tsi.getDatatype().getDatatype());

_query = "select schema_name, (select count(*) from tap_schema.alldatatypes) from tap_schema.tables";
_expected = "select schema_name, (select count(*) from tap_schema.alldatatypes) from tap_schema.tables";
selectList = doit();
assertTrue(selectList.size() == 2);
tsi = selectList.get(1);
assertEquals("count", tsi.getName().toLowerCase());
assertEquals("count2", tsi.getName().toLowerCase());
assertEquals("long", tsi.getDatatype().getDatatype());
}

Expand Down
38 changes: 27 additions & 11 deletions cadc-adql/src/test/java/ca/nrc/cadc/tap/parser/extractor/ExtractorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,8 +158,8 @@ private void doit(String query, List<TapSelectItem> expectedList, boolean isVali
List<TapSelectItem> selectList = _en.getSelectList();
for (int i = 0; i < expectedList.size(); i++)
{
TapSelectItem expected = expectedList.get(0);
TapSelectItem actual = selectList.get(0);
TapSelectItem expected = expectedList.get(i);
TapSelectItem actual = selectList.get(i);
log.debug("expected: " + expected);
log.debug("actual: " + actual);

Expand Down Expand Up @@ -236,7 +236,7 @@ public void testFunctionExpression()
String query = "select area(position_center_ra) from caom.siav1";

List<TapSelectItem> expectedList = new ArrayList<>();
TapSelectItem paramDesc = new TapSelectItem("area", TapDataType.DOUBLE);
TapSelectItem paramDesc = new TapSelectItem("area1", TapDataType.DOUBLE);
expectedList.add(paramDesc);

doit(query, expectedList);
Expand Down Expand Up @@ -264,7 +264,7 @@ public void testArgumentDatatypeFunctionExpression()
String query = "select max(position_center_ra) from caom.siav1";

List<TapSelectItem> expectedList = new ArrayList<>();
TapSelectItem paramDesc = new TapSelectItem("max", TapDataType.DOUBLE);
TapSelectItem paramDesc = new TapSelectItem("max1", TapDataType.DOUBLE);
expectedList.add(paramDesc);

doit(query, expectedList);
Expand All @@ -276,7 +276,7 @@ public void testNestedFunctionExpression()
String query = "select max(area(position_center_ra)) from caom.siav1";

List<TapSelectItem> expectedList = new ArrayList<>();
TapSelectItem paramDesc = new TapSelectItem("max", TapDataType.DOUBLE);
TapSelectItem paramDesc = new TapSelectItem("max1", TapDataType.DOUBLE);
expectedList.add(paramDesc);

doit(query, expectedList);
Expand All @@ -288,7 +288,7 @@ public void testNestedFunctionExpression2()
String query = "select area(max(position_center_ra)) from caom.siav1";

List<TapSelectItem> expectedList = new ArrayList<TapSelectItem>();
TapSelectItem paramDesc = new TapSelectItem("area", TapDataType.DOUBLE);
TapSelectItem paramDesc = new TapSelectItem("area1", TapDataType.DOUBLE);
expectedList.add(paramDesc);

doit(query, expectedList);
Expand All @@ -300,7 +300,7 @@ public void testRecursiveNestedFunctionExpression()
String query = "select max(min(avg(position_center_ra))) from caom.siav1";

List<TapSelectItem> expectedList = new ArrayList<>();
TapSelectItem paramDesc = new TapSelectItem("max", TapDataType.DOUBLE);
TapSelectItem paramDesc = new TapSelectItem("max1", TapDataType.DOUBLE);
expectedList.add(paramDesc);

doit(query, expectedList);
Expand All @@ -312,7 +312,7 @@ public void testConstantExpression()
String query = "select 1 from caom.siav1";

List<TapSelectItem> expectedList = new ArrayList<>();
TapSelectItem paramDesc = new TapSelectItem("1", new TapDataType("char", "*", null));
TapSelectItem paramDesc = new TapSelectItem("col1", new TapDataType("long", null, null));
expectedList.add(paramDesc);

doit(query, expectedList);
Expand All @@ -324,7 +324,7 @@ public void testConstantWithAliasExpression()
String query = "select 1 as one from caom.siav1";

List<TapSelectItem> expectedList = new ArrayList<TapSelectItem>();
TapSelectItem paramDesc = new TapSelectItem("one", new TapDataType("char", "*", null));
TapSelectItem paramDesc = new TapSelectItem("one", new TapDataType("long", null, null));
expectedList.add(paramDesc);

doit(query, expectedList);
Expand All @@ -336,7 +336,7 @@ public void testFunctionWithConstantExpression()
String query = "select max(1) from caom.siav1";

List<TapSelectItem> expectedList = new ArrayList<TapSelectItem>();
TapSelectItem paramDesc = new TapSelectItem("max", new TapDataType("char", "*", null));
TapSelectItem paramDesc = new TapSelectItem("max1", new TapDataType("long", null, null));
expectedList.add(paramDesc);

doit(query, expectedList);
Expand All @@ -348,10 +348,26 @@ public void testFunctionWithConstantAndAliasExpression()
String query = "select max(1) as foo from caom.siav1";

List<TapSelectItem> expectedList = new ArrayList<TapSelectItem>();
TapSelectItem paramDesc = new TapSelectItem("foo", new TapDataType("char", "*", null));
TapSelectItem paramDesc = new TapSelectItem("foo", new TapDataType("long", null, null));
expectedList.add(paramDesc);

doit(query, expectedList);
}

@Test
public void testQueryWithTwoFunctions()
{
String query = "select max(1), min(1), count(5) from caom.siav1";

List<TapSelectItem> expectedList = new ArrayList<TapSelectItem>();
TapSelectItem p1 = new TapSelectItem("max1", new TapDataType("long", null, null));
TapSelectItem p2 = new TapSelectItem("min2", new TapDataType("long", null, null));
TapSelectItem p3 = new TapSelectItem("count3", new TapDataType("long", null, null));
expectedList.add(p1);
expectedList.add(p2);
expectedList.add(p3);

doit(query, expectedList);
}

}

0 comments on commit 50a861a

Please sign in to comment.