Skip to content

Commit

Permalink
youcat: fix to use default PostgresqJobPersistence ctor
Browse files Browse the repository at this point in the history
  • Loading branch information
@pdowler
pdowler committed Aug 26, 2024
1 parent dc8f787 commit 6a9ac90
Show file tree
Hide file tree
Showing 7 changed files with 91 additions and 35 deletions.
2 changes: 1 addition & 1 deletion youcat/VERSION
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
## deployable containers have a semantic and build tag
# semantic version tag: major.minor
# build version tag: timestamp
VER=0.6.2
VER=0.6.3
TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")"
unset VER
2 changes: 1 addition & 1 deletion youcat/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ dependencies {
compile 'org.opencadc:cadc-dali:[1.2.12,)'
compile 'org.opencadc:cadc-adql:[1.1.14,)'
compile 'org.opencadc:cadc-uws:[1.0.2,)'
compile 'org.opencadc:cadc-uws-server:[1.2.18,)'
compile 'org.opencadc:cadc-uws-server:[1.2.22,)'
compile 'org.opencadc:cadc-tap-schema:[1.1.32,)'
compile 'org.opencadc:cadc-tap-server:[1.1.24,)'
compile 'org.opencadc:cadc-tap-server-pg:[1.1.0,)'
Expand Down
66 changes: 64 additions & 2 deletions youcat/src/intTest/java/org/opencadc/youcat/CatSyncUploadTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,22 @@
package org.opencadc.youcat;

import ca.nrc.cadc.conformance.uws2.JobResultWrapper;
import ca.nrc.cadc.dali.tables.votable.VOTableDocument;
import ca.nrc.cadc.dali.tables.votable.VOTableInfo;
import ca.nrc.cadc.dali.tables.votable.VOTableReader;
import ca.nrc.cadc.dali.tables.votable.VOTableResource;
import ca.nrc.cadc.tap.integration.TapSyncUploadTest;
import ca.nrc.cadc.tap.integration.VOTableHandler;
import ca.nrc.cadc.util.FileUtil;
import ca.nrc.cadc.util.Log4jInit;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.net.URI;
import java.util.HashMap;
import java.util.Map;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.Assert;
import org.junit.Test;

/**
*
Expand All @@ -17,7 +27,7 @@ public class CatSyncUploadTest extends TapSyncUploadTest {
private static final Logger log = Logger.getLogger(CatSyncUploadTest.class);

static {
Log4jInit.setLevel("ca.nrc.cadc.cat", Level.INFO);
Log4jInit.setLevel("org.opencadc.youcat", Level.INFO);
Log4jInit.setLevel("ca.nrc.cadc.tap", Level.INFO);
Log4jInit.setLevel("ca.nrc.cadc.conformance.uws2", Level.INFO);
}
Expand All @@ -28,4 +38,56 @@ public CatSyncUploadTest() {
setTestFile(f);
setTestURL(CatAsyncUploadTest.getVOTableURL(f));
}

@Test
public void testUploadBinary() {
try {
Log4jInit.setLevel("org.opencadc.youcat", Level.DEBUG);

File binFile = FileUtil.getFileFromResource("binary-vot.xml", CatSyncUploadTest.class);

String tableName = "mytab";
Map<String, Object> params = new HashMap<String, Object>();
params.put("upload1", binFile);
params.put("UPLOAD", tableName + ",param:upload1");
params.put("LANG", "ADQL");
params.put("QUERY", "select * from tap_upload." + tableName);

JobResultWrapper result = createAndExecuteSyncParamJobPOST("testUploadFile", params);

Assert.assertNull(result.throwable);
Assert.assertEquals(200, result.responseCode);

Assert.assertNotNull(result.syncOutput);
ByteArrayInputStream istream = new ByteArrayInputStream(result.syncOutput);
VOTableReader vrdr = new VOTableReader();
VOTableDocument vot = vrdr.read(istream);

String queryStatus = getQueryStatus(vot);
Assert.assertNotNull("QUERY_STATUS", queryStatus);
Assert.assertEquals("OK", queryStatus);

// TODO: verify round-trip of testFile1 -> vot?
} catch (Exception unexpected) {
log.error("unexpected exception", unexpected);
Assert.fail("unexpected exception: " + unexpected);
}
}

static String getQueryStatus(VOTableDocument vot) {
VOTableResource vr = vot.getResourceByType("results");
Assert.assertNotNull(vr);
log.debug("found resource: " + vr.getName() + " " + vr.getType());
String ret = null;
// find the last QUERY_STATUS and return that because there can be a trailing
// status when result processing fails
for (VOTableInfo vi : vr.getInfos()) {
if ("QUERY_STATUS".equals(vi.getName())) {
ret = vi.getValue();
log.warn("found status: " + ret);
}
}
log.warn("return status: " + ret);
return ret;
}
}
31 changes: 22 additions & 9 deletions youcat/src/intTest/java/org/opencadc/youcat/CreateTableTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ public void testCreateQueryDropVOSI() {
Assert.assertFalse("no result rows", iter.hasNext());

// cleanup on success
doDelete(schemaOwner, testTable, false);
//doDelete(schemaOwner, testTable, false);
} catch (Exception unexpected) {
log.error("unexpected exception", unexpected);
Assert.fail("unexpected exception: " + unexpected);
Expand All @@ -211,13 +211,26 @@ public void testCreateQueryDropVOTable() {
vtab.getFields().add(new VOTableField("c3", TapDataType.LONG.getDatatype()));
vtab.getFields().add(new VOTableField("c4", TapDataType.FLOAT.getDatatype()));
vtab.getFields().add(new VOTableField("c5", TapDataType.DOUBLE.getDatatype()));
VOTableField tsf = new VOTableField("c6", TapDataType.TIMESTAMP.getDatatype(), TapDataType.TIMESTAMP.arraysize);
tsf.xtype = TapDataType.TIMESTAMP.xtype;
vtab.getFields().add(tsf);
vtab.getFields().add(new VOTableField("e7", TapDataType.INTERVAL.getDatatype()));
vtab.getFields().add(new VOTableField("e8", TapDataType.POINT.getDatatype()));
vtab.getFields().add(new VOTableField("e9", TapDataType.CIRCLE.getDatatype()));
vtab.getFields().add(new VOTableField("e10", TapDataType.POLYGON.getDatatype()));

// extended types
VOTableField tf;
tf = new VOTableField("e6", TapDataType.TIMESTAMP.getDatatype(), TapDataType.TIMESTAMP.arraysize);
tf.xtype = TapDataType.TIMESTAMP.xtype;
vtab.getFields().add(tf);

tf = new VOTableField("e7", TapDataType.INTERVAL.getDatatype(), TapDataType.INTERVAL.arraysize);
tf.xtype = TapDataType.INTERVAL.xtype;
vtab.getFields().add(tf);
tf = new VOTableField("e8", TapDataType.POINT.getDatatype(), TapDataType.POINT.arraysize);
tf.xtype = TapDataType.POINT.xtype;
vtab.getFields().add(tf);
tf = new VOTableField("e9", TapDataType.CIRCLE.getDatatype(), TapDataType.CIRCLE.arraysize);
tf.xtype = TapDataType.CIRCLE.xtype;
vtab.getFields().add(tf);
tf = new VOTableField("e10", TapDataType.POLYGON.getDatatype(), TapDataType.POLYGON.arraysize);
tf.xtype = TapDataType.POLYGON.xtype;
vtab.getFields().add(tf);

// arrays
vtab.getFields().add(new VOTableField("a11", "short", "*"));
vtab.getFields().add(new VOTableField("a12", "int", "*"));
Expand Down Expand Up @@ -256,7 +269,7 @@ public void write(OutputStream out) throws IOException {
Assert.assertFalse("no result rows", iter.hasNext());


doDelete(schemaOwner, testTable, false);
//doDelete(schemaOwner, testTable, false);

} catch (Exception unexpected) {
log.error("unexpected exception", unexpected);
Expand Down
3 changes: 1 addition & 2 deletions youcat/src/main/java/org/opencadc/youcat/QueryJobManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,8 @@ public class QueryJobManager extends RequestPathJobManager {
public QueryJobManager() {
super();

IdentityManager im = AuthenticationUtil.getIdentityManager();
// persist UWS jobs to PostgreSQL using default jdbc/uws connection pool
JobPersistence jobPersist = new PostgresJobPersistence(new RandomStringGenerator(16), im, true);
JobPersistence jobPersist = new PostgresJobPersistence();

// max threads: 6 == number of simultaneously running async queries (per
// web server), plus sync queries, plus VOSI-tables queries
Expand Down
4 changes: 1 addition & 3 deletions youcat/src/main/java/org/opencadc/youcat/TableUpdateJobManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,7 @@ public class TableUpdateJobManager extends RequestPathJobManager {
public TableUpdateJobManager() {
super();

IdentityManager im = AuthenticationUtil.getIdentityManager();
// persist UWS jobs to PostgreSQL using default jdbc/uws connection pool
JobPersistence jobPersist = new PostgresJobPersistence(im);
JobPersistence jobPersist = new PostgresJobPersistence();

// max threads: 3 == number of simultaneously running jobs (per web server)
JobExecutor jobExec = new ThreadPoolExecutor(jobPersist, TableUpdateRunnerImpl.class, 3);
Expand Down
18 changes: 1 addition & 17 deletions youcat/src/main/java/org/opencadc/youcat/tap/UploadManagerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,26 +84,10 @@ public class UploadManagerImpl extends BasicUploadManager {
// 500k rows X 1 column (more if compact)
MAX_UPLOAD = new UploadLimits(32 * 1024L * 1024L); // 32 MiB
// TODO: columnLimit to prevent really wide tables?
MAX_UPLOAD.rowLimit = 100000; // sane batch size users can manage
//MAX_UPLOAD.rowLimit = 100000; // sane batch size users can manage
}

public UploadManagerImpl() {
super(MAX_UPLOAD);
}

/**
* Create the SQL to grant select privileges to the cvopub group for the
* specified table.
*
* @param databaseTableName fully qualified table name.
* @return SQL to grant select privileges to the cvopub group.
*/
@Override
protected String getGrantSelectTableSQL(String databaseTableName) {
StringBuilder sb = new StringBuilder();
sb.append("grant select on table ");
sb.append(databaseTableName);
sb.append(" to group cvopub");
return sb.toString();
}
}

0 comments on commit 6a9ac90

Please sign in to comment.