Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

cadc-tap-schema: add support for admin to create schema #168

Merged
merged 11 commits into from
Jul 19, 2024
Merged

cadc-tap-schema: add support for admin to create schema #168

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
8511e59
cadc-tap-schema: add support for admin to create schema
pdowler Jul 17, 2024
0f142c7
youcat: configure admin account
pdowler Jul 17, 2024
0ed4550
youcat: add intTest README
pdowler Jul 17, 2024
c36ed85
fix imports
pdowler Jul 17, 2024
b4131dd
cadc-tap-schema: improve batch inserts
pdowler Jul 18, 2024
3ccfcbf
fix bad import
pdowler Jul 18, 2024
e45731b
youcat: fix init to make admin fully optional
pdowler Jul 18, 2024
6ad30dd
Update youcat/src/intTest/README.md
pdowler Jul 19, 2024
dbb806b
Update youcat/src/intTest/README.md
pdowler Jul 19, 2024
0b5c087
clarify adminUser in README
pdowler Jul 19, 2024
dc8f787
update copright date and clean imports
pdowler Jul 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cadc-tap-schema/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ sourceCompatibility = 1.8

group = 'org.opencadc'

version = '1.1.32'
version = '1.1.33'

description = 'OpenCADC TAP-1.1 tap schema server library'
def git_url = 'https://github.com/opencadc/tap'
Expand Down
139 changes: 75 additions & 64 deletions cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/TableLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
******************* CANADIAN ASTRONOMY DATA CENTRE *******************
************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
*
* (c) 2018. (c) 2018.
* (c) 2024. (c) 2024.
* Government of Canada Gouvernement du Canada
* National Research Council Conseil national de recherches
* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
Expand Down Expand Up @@ -81,10 +81,10 @@
import ca.nrc.cadc.tap.schema.TableDesc;
import ca.nrc.cadc.tap.schema.TapDataType;
import java.net.URI;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.Iterator;
Expand All @@ -94,7 +94,7 @@
import org.opencadc.tap.io.InconsistentTableDataException;
import org.opencadc.tap.io.TableDataInputStream;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.PreparedStatementCreator;
import org.springframework.jdbc.core.ParameterizedPreparedStatementSetter;

/**
* Utility to bulk load content into a table.
Expand Down Expand Up @@ -142,25 +142,29 @@ public void load(TableDesc destTable, TableDataInputStream data) {
Iterator<List<Object>> dataIterator = data.iterator();
List<Object> nextRow = null;

List<List<Object>> batch = new ArrayList<>(batchSize);
int count = 0;
try {
while (!done) {
count = 0;
tm.startTransaction();
prof.checkpoint("start-transaction");
BulkInsertStatement bulkInsertStatement = new BulkInsertStatement(reorgTable);

while (count < batchSize && dataIterator.hasNext()) {
while (batch.size() < batchSize && dataIterator.hasNext()) {
nextRow = dataIterator.next();
convertValueObjects(nextRow);
jdbc.update(sql, nextRow.toArray());
batch.add(nextRow);
count++;
}
log.debug("Inserting " + count + " rows in this batch.");
log.debug("Inserting " + batch.size() + " rows in this batch.");
jdbc.batchUpdate(sql, batch, batchSize, bulkInsertStatement);
prof.checkpoint("batch-of-inserts");

tm.commitTransaction();
prof.checkpoint("commit-transaction");
totalInserts += count;
totalInserts += batch.size();
batch.clear();
done = !dataIterator.hasNext();
}
} catch (IllegalArgumentException | IndexOutOfBoundsException | InconsistentTableDataException ex) {
Expand All @@ -178,8 +182,8 @@ public void load(TableDesc destTable, TableDataInputStream data) {
} catch (Exception oops) {
log.error("Unexpected: could not rollback transaction", oops);
}
throw new IllegalArgumentException("Inserted " + totalInserts + " rows. " +
"Current batch failed with: " + ex.getMessage() + " on line " + (totalInserts + count));
throw new IllegalArgumentException("Inserted " + totalInserts + " rows."
+ " Current batch failed with: " + ex.getMessage() + " on line " + (totalInserts + count));
} catch (Throwable t) {
try {
data.close();
Expand All @@ -197,8 +201,8 @@ public void load(TableDesc destTable, TableDataInputStream data) {
}

log.debug("Batch insert failure", t);
throw new RuntimeException("Inserted " + totalInserts + " rows. "
+ "Current batch of " + batchSize + " failed with: " + t.getMessage(), t);
throw new RuntimeException("Inserted " + totalInserts + " rows."
+ " Current batch of " + batchSize + " failed with: " + t.getMessage(), t);

} finally {
if (tm.isOpen()) {
Expand Down Expand Up @@ -241,27 +245,26 @@ private String generateInsertSQL(TableDesc td) {
return sb.toString();
}

private class BulkInsertStatement implements PreparedStatementCreator {
private class BulkInsertStatement implements ParameterizedPreparedStatementSetter<List<Object>> {
private final Calendar utc = Calendar.getInstance(DateUtil.UTC);
private TableDesc tableDesc;
Object[] row;

public BulkInsertStatement(TableDesc tableDesc) {
this.tableDesc = tableDesc;
}

@Override
public PreparedStatement createPreparedStatement(Connection con) throws SQLException {
String sql = generateInsertSQL(tableDesc);
PreparedStatement ret = con.prepareStatement(sql);

for (int i = 0; i < tableDesc.getColumnDescs().size(); i++) {
ColumnDesc cd = tableDesc.getColumnDescs().get(i);
Object val = row[i];
public void setValues(PreparedStatement ps, List<Object> row) throws SQLException {
int col = 1;
for (Object val : row) {
ColumnDesc cd = tableDesc.getColumnDescs().get(col - 1);
if (val != null && val instanceof Date && TapDataType.TIMESTAMP.equals(cd.getDatatype())) {
Date d = (Date) val;
ret.setTimestamp(i + 1, new Timestamp(d.getTime()), utc);
ps.setTimestamp(col++, new Timestamp(d.getTime()), utc);
} else {
ret.setObject(i + 1, val);
ps.setObject(col++, val);
}
}
return ret;
}
}

Expand All @@ -274,52 +277,60 @@ public long getTotalInserts() {

// convert values that the JDBC driver won't accept
private void convertValueObjects(List<Object> values) {
for (int i=0; i < values.size(); i++) {
for (int i = 0; i < values.size(); i++) {
Object v = values.get(i);
if (v != null) {
if (v instanceof URI) {
String nv = ((URI) v).toASCIIString();
values.set(i, nv);
} else if (v instanceof DoubleInterval) {
Object nv = ddType.getIntervalObject((DoubleInterval) v);
values.set(i, nv);
} else if (v instanceof LongInterval) {
Interval inter = (Interval) v;
DoubleInterval di = new DoubleInterval(inter.getLower().doubleValue(), inter.getUpper().doubleValue());
Object nv = ddType.getIntervalObject(di);
values.set(i, nv);
} else if (v instanceof Point) {
Object nv = ddType.getPointObject((Point) v);
values.set(i, nv);
} else if (v instanceof Circle) {
Object nv = ddType.getCircleObject((Circle) v);
values.set(i, nv);
} else if (v instanceof Polygon) {
Object nv = ddType.getPolygonObject((Polygon) v);
values.set(i, nv);
} else if (v instanceof ca.nrc.cadc.stc.Position) {
Object nv = ddType.getPointObject((ca.nrc.cadc.stc.Position) v);
values.set(i, nv);
} else if (v instanceof ca.nrc.cadc.stc.Region) {
Object nv = ddType.getRegionObject((ca.nrc.cadc.stc.Region) v);
values.set(i, nv);
} else if (v instanceof short[]) {
Object nv = ddType.getArrayObject((short[]) v);
values.set(i, nv);
} else if (v instanceof int[]) {
Object nv = ddType.getArrayObject((int[]) v);
values.set(i, nv);
} else if (v instanceof long[]) {
Object nv = ddType.getArrayObject((long[]) v);
values.set(i, nv);
} else if (v instanceof float[]) {
Object nv = ddType.getArrayObject((float[]) v);
values.set(i, nv);
} else if (v instanceof double[]) {
Object nv = ddType.getArrayObject((double[]) v);
Object nv = convertValueObject(v);
if (v != nv) {
values.set(i, nv);
}
}
}
}

private Object convertValueObject(Object v) {
if (v instanceof URI) {
return ((URI) v).toASCIIString();
}
if (v instanceof DoubleInterval) {
return ddType.getIntervalObject((DoubleInterval) v);
}
if (v instanceof LongInterval) {
Interval inter = (Interval) v;
DoubleInterval di = new DoubleInterval(inter.getLower().doubleValue(), inter.getUpper().doubleValue());
return ddType.getIntervalObject(di);
}
if (v instanceof Point) {
return ddType.getPointObject((Point) v);
}
if (v instanceof Circle) {
return ddType.getCircleObject((Circle) v);
}
if (v instanceof Polygon) {
return ddType.getPolygonObject((Polygon) v);
}
if (v instanceof ca.nrc.cadc.stc.Position) {
return ddType.getPointObject((ca.nrc.cadc.stc.Position) v);
}
if (v instanceof ca.nrc.cadc.stc.Region) {
return ddType.getRegionObject((ca.nrc.cadc.stc.Region) v);
}
if (v instanceof short[]) {
return ddType.getArrayObject((short[]) v);
}
if (v instanceof int[]) {
return ddType.getArrayObject((int[]) v);
}
if (v instanceof long[]) {
return ddType.getArrayObject((long[]) v);
}
if (v instanceof float[]) {
return ddType.getArrayObject((float[]) v);
}
if (v instanceof double[]) {
return ddType.getArrayObject((double[]) v);
}

return v;
}
}
33 changes: 31 additions & 2 deletions cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/GetAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@

import ca.nrc.cadc.net.ResourceNotFoundException;
import ca.nrc.cadc.rest.RestAction;
import ca.nrc.cadc.tap.schema.SchemaDesc;
import ca.nrc.cadc.tap.schema.TableDesc;
import ca.nrc.cadc.tap.schema.TapSchema;
import ca.nrc.cadc.tap.schema.TapSchemaDAO;
Expand All @@ -93,8 +94,14 @@ public GetAction() {

@Override
public void doAction() throws Exception {
String tableName = getTableName();
log.debug("GET: " + tableName);
String schemaName = null;
String tableName = null;
String[] target = getTarget();
if (target != null) {
schemaName = target[0];
tableName = target[1];
}
log.debug("GET: " + schemaName + " " + tableName);

if (!readable) {
throw new AccessControlException(RestAction.STATE_OFFLINE_MSG);
Expand All @@ -118,13 +125,35 @@ public void doAction() throws Exception {
if (tableName != null) {
checkTableReadPermissions(dao, tableName);
TableDesc td = dao.getTable(tableName);
if (td == null) {
// currently, permission check already threw this
throw new ResourceNotFoundException("table not found: " + tableName);
}
TableWriter tw = new TableWriter();
syncOutput.setCode(HttpServletResponse.SC_OK);
syncOutput.setHeader("Content-Type", "text/xml");
tw.write(td, new OutputStreamWriter(syncOutput.getOutputStream()));
} else if (schemaName != null) {
checkViewSchemaPermissions(dao, schemaName);
// TODO: TapSchemaDAO only supports schema only, ok for detail=min
// should at least list tables for default detail
// should provide columns at detail=max
SchemaDesc sd = dao.getSchema(schemaName, (depth == TapSchemaDAO.MIN_DEPTH));
if (sd == null) {
// currently, permission check already threw this
throw new ResourceNotFoundException("schema not found: " + schemaName);
}
TapSchema tapSchema = new TapSchema();
tapSchema.getSchemaDescs().add(sd);

TableSetWriter tsw = new TableSetWriter();
syncOutput.setCode(HttpServletResponse.SC_OK);
syncOutput.setHeader("Content-Type", "text/xml");
tsw.write(tapSchema, new OutputStreamWriter(syncOutput.getOutputStream()));
} else {
TapSchemaLoader loader = new TapSchemaLoader(dao);
TapSchema tapSchema = loader.load(depth);

TableSetWriter tsw = new TableSetWriter();
syncOutput.setCode(HttpServletResponse.SC_OK);
syncOutput.setHeader("Content-Type", "text/xml");
Expand Down
26 changes: 12 additions & 14 deletions cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/GetPermissionsAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,20 +67,15 @@

package ca.nrc.cadc.vosi.actions;

import ca.nrc.cadc.rest.InlineContentHandler;
import ca.nrc.cadc.tap.schema.TapPermissions;
import ca.nrc.cadc.tap.schema.TapSchemaDAO;
import java.io.OutputStream;

import javax.security.auth.Subject;
import javax.security.auth.x500.X500Principal;

import org.apache.log4j.Logger;
import org.opencadc.gms.GroupURI;

import ca.nrc.cadc.auth.HttpPrincipal;
import ca.nrc.cadc.auth.NumericPrincipal;
import ca.nrc.cadc.rest.InlineContentHandler;
import ca.nrc.cadc.tap.schema.TapPermissions;
import ca.nrc.cadc.tap.schema.TapSchemaDAO;

/**
* Return the permissions for the object identified by the 'name'
* parameter.
Expand All @@ -94,15 +89,18 @@ public class GetPermissionsAction extends TablesAction {

@Override
public void doAction() throws Exception {
String name = getTableName();
log.debug("POST: " + name);
String[] target = getTarget();
if (target == null) {
throw new IllegalArgumentException("no schema|table name in path");
}
String name = target[0]; // schema
if (target[1] != null) {
name = target[1]; // table
}
log.debug("name: " + name);

checkWritable();

if (name == null) {
throw new IllegalArgumentException( "Missing param: name");
}

TapSchemaDAO dao = getTapSchemaDAO();
TapPermissions permissions = null;
if (Util.isSchemaName(name)) {
Expand Down
13 changes: 8 additions & 5 deletions cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/PostPermissionsAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,12 +101,15 @@ public PostPermissionsAction() {

@Override
public void doAction() throws Exception {

String name = getTableName();
log.debug("name: " + name);
if (name == null) {
throw new IllegalArgumentException("Missing param: name");
String[] target = getTarget();
if (target == null) {
throw new IllegalArgumentException("no schema|table name in path");
}
String name = target[0]; // schema
if (target[1] != null) {
name = target[1]; // table
}
log.debug("name: " + name);

checkWritable();

Expand Down
Loading
Loading