Skip to content

Commit

Permalink
Merge pull request #168 from pdowler/main
Browse files Browse the repository at this point in the history 
cadc-tap-schema: add support for admin to create schema
  • Loading branch information
@pdowler
pdowler authored Jul 19, 2024
2 parents ffde32e + dc8f787 commit c5c3235
Show file tree
Hide file tree
Showing 16 changed files with 604 additions and 296 deletions.
2 changes: 1 addition & 1 deletion cadc-tap-schema/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ sourceCompatibility = 1.8

group = 'org.opencadc'

version = '1.1.32'
version = '1.1.33'

description = 'OpenCADC TAP-1.1 tap schema server library'
def git_url = 'https://github.com/opencadc/tap'
Expand Down
139 changes: 75 additions & 64 deletions cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/TableLoader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
******************* CANADIAN ASTRONOMY DATA CENTRE *******************
************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
*
* (c) 2018. (c) 2018.
* (c) 2024. (c) 2024.
* Government of Canada Gouvernement du Canada
* National Research Council Conseil national de recherches
* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
Expand Down Expand Up @@ -81,10 +81,10 @@
import ca.nrc.cadc.tap.schema.TableDesc;
import ca.nrc.cadc.tap.schema.TapDataType;
import java.net.URI;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.Iterator;
Expand All @@ -94,7 +94,7 @@
import org.opencadc.tap.io.InconsistentTableDataException;
import org.opencadc.tap.io.TableDataInputStream;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.PreparedStatementCreator;
import org.springframework.jdbc.core.ParameterizedPreparedStatementSetter;

/**
* Utility to bulk load content into a table.
Expand Down Expand Up @@ -142,25 +142,29 @@ public void load(TableDesc destTable, TableDataInputStream data) {
Iterator<List<Object>> dataIterator = data.iterator();
List<Object> nextRow = null;

List<List<Object>> batch = new ArrayList<>(batchSize);
int count = 0;
try {
while (!done) {
count = 0;
tm.startTransaction();
prof.checkpoint("start-transaction");
BulkInsertStatement bulkInsertStatement = new BulkInsertStatement(reorgTable);

while (count < batchSize && dataIterator.hasNext()) {
while (batch.size() < batchSize && dataIterator.hasNext()) {
nextRow = dataIterator.next();
convertValueObjects(nextRow);
jdbc.update(sql, nextRow.toArray());
batch.add(nextRow);
count++;
}
log.debug("Inserting " + count + " rows in this batch.");
log.debug("Inserting " + batch.size() + " rows in this batch.");
jdbc.batchUpdate(sql, batch, batchSize, bulkInsertStatement);
prof.checkpoint("batch-of-inserts");

tm.commitTransaction();
prof.checkpoint("commit-transaction");
totalInserts += count;
totalInserts += batch.size();
batch.clear();
done = !dataIterator.hasNext();
}
} catch (IllegalArgumentException | IndexOutOfBoundsException | InconsistentTableDataException ex) {
Expand All @@ -178,8 +182,8 @@ public void load(TableDesc destTable, TableDataInputStream data) {
} catch (Exception oops) {
log.error("Unexpected: could not rollback transaction", oops);
}
throw new IllegalArgumentException("Inserted " + totalInserts + " rows. " +
"Current batch failed with: " + ex.getMessage() + " on line " + (totalInserts + count));
throw new IllegalArgumentException("Inserted " + totalInserts + " rows."
+ " Current batch failed with: " + ex.getMessage() + " on line " + (totalInserts + count));
} catch (Throwable t) {
try {
data.close();
Expand All @@ -197,8 +201,8 @@ public void load(TableDesc destTable, TableDataInputStream data) {
}

log.debug("Batch insert failure", t);
throw new RuntimeException("Inserted " + totalInserts + " rows. "
+ "Current batch of " + batchSize + " failed with: " + t.getMessage(), t);
throw new RuntimeException("Inserted " + totalInserts + " rows."
+ " Current batch of " + batchSize + " failed with: " + t.getMessage(), t);

} finally {
if (tm.isOpen()) {
Expand Down Expand Up @@ -241,27 +245,26 @@ private String generateInsertSQL(TableDesc td) {
return sb.toString();
}

private class BulkInsertStatement implements PreparedStatementCreator {
private class BulkInsertStatement implements ParameterizedPreparedStatementSetter<List<Object>> {
private final Calendar utc = Calendar.getInstance(DateUtil.UTC);
private TableDesc tableDesc;
Object[] row;

public BulkInsertStatement(TableDesc tableDesc) {
this.tableDesc = tableDesc;
}

@Override
public PreparedStatement createPreparedStatement(Connection con) throws SQLException {
String sql = generateInsertSQL(tableDesc);
PreparedStatement ret = con.prepareStatement(sql);

for (int i = 0; i < tableDesc.getColumnDescs().size(); i++) {
ColumnDesc cd = tableDesc.getColumnDescs().get(i);
Object val = row[i];
public void setValues(PreparedStatement ps, List<Object> row) throws SQLException {
int col = 1;
for (Object val : row) {
ColumnDesc cd = tableDesc.getColumnDescs().get(col - 1);
if (val != null && val instanceof Date && TapDataType.TIMESTAMP.equals(cd.getDatatype())) {
Date d = (Date) val;
ret.setTimestamp(i + 1, new Timestamp(d.getTime()), utc);
ps.setTimestamp(col++, new Timestamp(d.getTime()), utc);
} else {
ret.setObject(i + 1, val);
ps.setObject(col++, val);
}
}
return ret;
}
}

Expand All @@ -274,52 +277,60 @@ public long getTotalInserts() {

// convert values that the JDBC driver won't accept
private void convertValueObjects(List<Object> values) {
for (int i=0; i < values.size(); i++) {
for (int i = 0; i < values.size(); i++) {
Object v = values.get(i);
if (v != null) {
if (v instanceof URI) {
String nv = ((URI) v).toASCIIString();
values.set(i, nv);
} else if (v instanceof DoubleInterval) {
Object nv = ddType.getIntervalObject((DoubleInterval) v);
values.set(i, nv);
} else if (v instanceof LongInterval) {
Interval inter = (Interval) v;
DoubleInterval di = new DoubleInterval(inter.getLower().doubleValue(), inter.getUpper().doubleValue());
Object nv = ddType.getIntervalObject(di);
values.set(i, nv);
} else if (v instanceof Point) {
Object nv = ddType.getPointObject((Point) v);
values.set(i, nv);
} else if (v instanceof Circle) {
Object nv = ddType.getCircleObject((Circle) v);
values.set(i, nv);
} else if (v instanceof Polygon) {
Object nv = ddType.getPolygonObject((Polygon) v);
values.set(i, nv);
} else if (v instanceof ca.nrc.cadc.stc.Position) {
Object nv = ddType.getPointObject((ca.nrc.cadc.stc.Position) v);
values.set(i, nv);
} else if (v instanceof ca.nrc.cadc.stc.Region) {
Object nv = ddType.getRegionObject((ca.nrc.cadc.stc.Region) v);
values.set(i, nv);
} else if (v instanceof short[]) {
Object nv = ddType.getArrayObject((short[]) v);
values.set(i, nv);
} else if (v instanceof int[]) {
Object nv = ddType.getArrayObject((int[]) v);
values.set(i, nv);
} else if (v instanceof long[]) {
Object nv = ddType.getArrayObject((long[]) v);
values.set(i, nv);
} else if (v instanceof float[]) {
Object nv = ddType.getArrayObject((float[]) v);
values.set(i, nv);
} else if (v instanceof double[]) {
Object nv = ddType.getArrayObject((double[]) v);
Object nv = convertValueObject(v);
if (v != nv) {
values.set(i, nv);
}
}
}
}

private Object convertValueObject(Object v) {
if (v instanceof URI) {
return ((URI) v).toASCIIString();
}
if (v instanceof DoubleInterval) {
return ddType.getIntervalObject((DoubleInterval) v);
}
if (v instanceof LongInterval) {
Interval inter = (Interval) v;
DoubleInterval di = new DoubleInterval(inter.getLower().doubleValue(), inter.getUpper().doubleValue());
return ddType.getIntervalObject(di);
}
if (v instanceof Point) {
return ddType.getPointObject((Point) v);
}
if (v instanceof Circle) {
return ddType.getCircleObject((Circle) v);
}
if (v instanceof Polygon) {
return ddType.getPolygonObject((Polygon) v);
}
if (v instanceof ca.nrc.cadc.stc.Position) {
return ddType.getPointObject((ca.nrc.cadc.stc.Position) v);
}
if (v instanceof ca.nrc.cadc.stc.Region) {
return ddType.getRegionObject((ca.nrc.cadc.stc.Region) v);
}
if (v instanceof short[]) {
return ddType.getArrayObject((short[]) v);
}
if (v instanceof int[]) {
return ddType.getArrayObject((int[]) v);
}
if (v instanceof long[]) {
return ddType.getArrayObject((long[]) v);
}
if (v instanceof float[]) {
return ddType.getArrayObject((float[]) v);
}
if (v instanceof double[]) {
return ddType.getArrayObject((double[]) v);
}

return v;
}
}
35 changes: 32 additions & 3 deletions cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/GetAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
******************* CANADIAN ASTRONOMY DATA CENTRE *******************
************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
*
* (c) 2018. (c) 2018.
* (c) 2024. (c) 2024.
* Government of Canada Gouvernement du Canada
* National Research Council Conseil national de recherches
* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
Expand Down Expand Up @@ -69,6 +69,7 @@

import ca.nrc.cadc.net.ResourceNotFoundException;
import ca.nrc.cadc.rest.RestAction;
import ca.nrc.cadc.tap.schema.SchemaDesc;
import ca.nrc.cadc.tap.schema.TableDesc;
import ca.nrc.cadc.tap.schema.TapSchema;
import ca.nrc.cadc.tap.schema.TapSchemaDAO;
Expand All @@ -93,8 +94,14 @@ public GetAction() {

@Override
public void doAction() throws Exception {
String tableName = getTableName();
log.debug("GET: " + tableName);
String schemaName = null;
String tableName = null;
String[] target = getTarget();
if (target != null) {
schemaName = target[0];
tableName = target[1];
}
log.debug("GET: " + schemaName + " " + tableName);

if (!readable) {
throw new AccessControlException(RestAction.STATE_OFFLINE_MSG);
Expand All @@ -118,13 +125,35 @@ public void doAction() throws Exception {
if (tableName != null) {
checkTableReadPermissions(dao, tableName);
TableDesc td = dao.getTable(tableName);
if (td == null) {
// currently, permission check already threw this
throw new ResourceNotFoundException("table not found: " + tableName);
}
TableWriter tw = new TableWriter();
syncOutput.setCode(HttpServletResponse.SC_OK);
syncOutput.setHeader("Content-Type", "text/xml");
tw.write(td, new OutputStreamWriter(syncOutput.getOutputStream()));
} else if (schemaName != null) {
checkViewSchemaPermissions(dao, schemaName);
// TODO: TapSchemaDAO only supports schema only, ok for detail=min
// should at least list tables for default detail
// should provide columns at detail=max
SchemaDesc sd = dao.getSchema(schemaName, (depth == TapSchemaDAO.MIN_DEPTH));
if (sd == null) {
// currently, permission check already threw this
throw new ResourceNotFoundException("schema not found: " + schemaName);
}
TapSchema tapSchema = new TapSchema();
tapSchema.getSchemaDescs().add(sd);

TableSetWriter tsw = new TableSetWriter();
syncOutput.setCode(HttpServletResponse.SC_OK);
syncOutput.setHeader("Content-Type", "text/xml");
tsw.write(tapSchema, new OutputStreamWriter(syncOutput.getOutputStream()));
} else {
TapSchemaLoader loader = new TapSchemaLoader(dao);
TapSchema tapSchema = loader.load(depth);

TableSetWriter tsw = new TableSetWriter();
syncOutput.setCode(HttpServletResponse.SC_OK);
syncOutput.setHeader("Content-Type", "text/xml");
Expand Down
28 changes: 13 additions & 15 deletions cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/GetPermissionsAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
******************* CANADIAN ASTRONOMY DATA CENTRE *******************
************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
*
* (c) 2018. (c) 2018.
* (c) 2024. (c) 2024.
* Government of Canada Gouvernement du Canada
* National Research Council Conseil national de recherches
* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
Expand Down Expand Up @@ -67,20 +67,15 @@

package ca.nrc.cadc.vosi.actions;

import ca.nrc.cadc.rest.InlineContentHandler;
import ca.nrc.cadc.tap.schema.TapPermissions;
import ca.nrc.cadc.tap.schema.TapSchemaDAO;
import java.io.OutputStream;

import javax.security.auth.Subject;
import javax.security.auth.x500.X500Principal;

import org.apache.log4j.Logger;
import org.opencadc.gms.GroupURI;

import ca.nrc.cadc.auth.HttpPrincipal;
import ca.nrc.cadc.auth.NumericPrincipal;
import ca.nrc.cadc.rest.InlineContentHandler;
import ca.nrc.cadc.tap.schema.TapPermissions;
import ca.nrc.cadc.tap.schema.TapSchemaDAO;

/**
* Return the permissions for the object identified by the 'name'
* parameter.
Expand All @@ -94,15 +89,18 @@ public class GetPermissionsAction extends TablesAction {

@Override
public void doAction() throws Exception {
String name = getTableName();
log.debug("POST: " + name);
String[] target = getTarget();
if (target == null) {
throw new IllegalArgumentException("no schema|table name in path");
}
String name = target[0]; // schema
if (target[1] != null) {
name = target[1]; // table
}
log.debug("name: " + name);

checkWritable();

if (name == null) {
throw new IllegalArgumentException( "Missing param: name");
}

TapSchemaDAO dao = getTapSchemaDAO();
TapPermissions permissions = null;
if (Util.isSchemaName(name)) {
Expand Down
Loading

0 comments on commit c5c3235

Please sign in to comment.