diff --git a/argus/README.md b/argus/README.md index ddb21b1..64d3f8f 100644 --- a/argus/README.md +++ b/argus/README.md @@ -71,7 +71,16 @@ using the _TempStorageManager_, the base URL must include "/results" as the last (e.g. `https://example.net/argus/results`). ### argus.properties -This file may be needed in the future but is not currently used. +This file is currently optional and configures optional service features. +``` +# enable materialized views (optional; default: false) +org.opencadc.argus.enableMaterializedViews = true | false +``` + +The _enableMaterializedViews_ key causes `argus` to add descriptions of materialized views to the TAP_SCHEMA +for use in queries. NOTE: enabling this feature does not currrently create or maintain the views so only enable +this if you have out-of-band creation working. TODO: implement creation and periodic update of materialized +views in `argus`. ### injecting VOTable resources into query results Some columns in the CAOM metadata have _ID_ values that are asssigned to the VOTable FIELD in the output. diff --git a/argus/VERSION b/argus/VERSION index 8f38604..cb6c06c 100644 --- a/argus/VERSION +++ b/argus/VERSION @@ -1,6 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor # build version tag: timestamp -VER=1.0.8 +VER=1.0.9 TAGS="${VER} ${VER}-$(date -u +"%Y%m%dT%H%M%S")" unset VER diff --git a/argus/src/main/java/org/opencadc/argus/ArgusInitAction.java b/argus/src/main/java/org/opencadc/argus/ArgusInitAction.java index b5fab3b..c436525 100644 --- a/argus/src/main/java/org/opencadc/argus/ArgusInitAction.java +++ b/argus/src/main/java/org/opencadc/argus/ArgusInitAction.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2023. (c) 2023. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -71,6 +71,8 @@ import ca.nrc.cadc.rest.InitAction; import ca.nrc.cadc.tap.impl.InitCaomTapSchemaContent; import ca.nrc.cadc.tap.schema.InitDatabaseTS; +import ca.nrc.cadc.util.MultiValuedProperties; +import ca.nrc.cadc.util.PropertiesReader; import ca.nrc.cadc.uws.server.impl.InitDatabaseUWS; import javax.sql.DataSource; import org.apache.log4j.Logger; @@ -84,7 +86,22 @@ public class ArgusInitAction extends InitAction { private static final Logger log = Logger.getLogger(ArgusInitAction.class); + private static final String CONFIG_ENABLE_MV = "org.opencadc.argus.enableMaterializedViews"; + + private final boolean enableMaterialisedViews; + public ArgusInitAction() { + PropertiesReader pr = new PropertiesReader("argus.properties"); + boolean enableMV = false; + try { + MultiValuedProperties mvp = pr.getAllProperties(); + + String str = mvp.getFirstPropertyValue(CONFIG_ENABLE_MV); + enableMV = "true".equals(str); + } catch (Exception ex) { + log.warn("failed to read otional config: " + ex); + } + this.enableMaterialisedViews = enableMV; } @Override @@ -106,7 +123,7 @@ public void doInit() { // caom2 tap_schema content log.info("InitCaomTapSchemaContent: START"); - InitCaomTapSchemaContent lsc = new InitCaomTapSchemaContent(tapadm, null, "tap_schema"); + InitCaomTapSchemaContent lsc = new InitCaomTapSchemaContent(tapadm, null, "tap_schema", enableMaterialisedViews); lsc.doInit(); log.info("InitCaomTapSchemaContent: OK"); } catch (Exception ex) { diff --git a/caom2-tap-server/build.gradle b/caom2-tap-server/build.gradle index 597487a..8a24629 100644 --- a/caom2-tap-server/build.gradle +++ b/caom2-tap-server/build.gradle @@ -17,7 +17,7 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -version = '1.2.12' +version = '1.2.13' description = 'OpenCADC CAOM2 TAP server library' def git_url = 'https://github.com/opencadc/caom2service' diff --git a/caom2-tap-server/src/main/java/ca/nrc/cadc/tap/impl/InitCaomTapSchemaContent.java b/caom2-tap-server/src/main/java/ca/nrc/cadc/tap/impl/InitCaomTapSchemaContent.java index b681584..9943006 100644 --- a/caom2-tap-server/src/main/java/ca/nrc/cadc/tap/impl/InitCaomTapSchemaContent.java +++ b/caom2-tap-server/src/main/java/ca/nrc/cadc/tap/impl/InitCaomTapSchemaContent.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2023. (c) 2023. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -87,17 +87,18 @@ public class InitCaomTapSchemaContent extends InitDatabase { private static final Logger log = Logger.getLogger(InitCaomTapSchemaContent.class); public static final String MODEL_NAME = "caom-schema"; - public static final String MODEL_VERSION = "1.2.12"; + public static final String MODEL_VERSION = "1.2.13"; // the SQL is tightly coupled to cadc-tap-schema table names (for TAP-1.1) - static String[] CREATE_SQL = new String[] { + static String[] BASE_SQL = new String[] { "caom2.tap_schema_content11.sql", "ivoa.tap_schema_content11.sql" }; // upgrade is normally the same as create since SQL is idempotent - static String[] UPGRADE_SQL = new String[] { + static String[] BASE_EXTRA_SQL = new String[] { "caom2.tap_schema_content11.sql", + "caom2.tap_schema_content11-extra.sql", "ivoa.tap_schema_content11.sql" }; @@ -108,13 +109,18 @@ public class InitCaomTapSchemaContent extends InitDatabase { * @param dataSource connection with write permission to tap_schema tables * @param database database name (should be null if not needed in SQL) * @param schema schema name (usually tap_schema) + * @param extras add tap_schema content for extra tables (materialised views) */ - public InitCaomTapSchemaContent(DataSource dataSource, String database, String schema) { - super(dataSource, database, schema, MODEL_NAME, MODEL_VERSION); - for (String s : CREATE_SQL) { - createSQL.add(s); + public InitCaomTapSchemaContent(DataSource dataSource, String database, String schema, boolean extras) { + // use MODELVERSION/extras so changing extras will cause a recreate + // eg 1.2.13/false <-> 1.2.13/true + super(dataSource, database, schema, MODEL_NAME, MODEL_VERSION + "/" + extras); + String[] src = BASE_SQL; + if (extras) { + src = BASE_EXTRA_SQL; } - for (String s : UPGRADE_SQL) { + for (String s : src) { + createSQL.add(s); upgradeSQL.add(s); } } diff --git a/caom2-tap-server/src/main/resources/sql/caom2.tap_schema_content11-extra.sql b/caom2-tap-server/src/main/resources/sql/caom2.tap_schema_content11-extra.sql index 75597e0..e1b896b 100644 --- a/caom2-tap-server/src/main/resources/sql/caom2.tap_schema_content11-extra.sql +++ b/caom2-tap-server/src/main/resources/sql/caom2.tap_schema_content11-extra.sql @@ -2,11 +2,11 @@ insert into tap_schema.tables11 (schema_name,table_name,table_type,description,utype,table_index) values -- index start at 30 -( 'caom2', 'caom2.EnumField', 'view', 'pre-computed aggregate (group by) materialised view built from enumerated types in CAOM model', NULL , 30), -( 'caom2', 'caom2.ObsCoreEnumField', 'view', 'pre-computed aggregate (group by) materialised view built from enumerated types in ObsCore-1.1 model', NULL, 31), -( 'caom2', 'caom2.distinct_proposal_id', 'view', 'pre-computed materialised view of distinct caom2.Observation.proposal_id values', NULL , 32), -( 'caom2', 'caom2.distinct_proposal_pi', 'view', 'pre-computed materialised view of distinct caom2.Observation.proposal_pi values', NULL , 33), -( 'caom2', 'caom2.distinct_proposal_title', 'view', 'pre-computed materialised view of distinct caom2.Observation.proposal_title values', NULL , 34) +( 'caom2', 'caom2.EnumField', 'table', 'pre-computed aggregate (group by) materialised view built from enumerated types in CAOM model', NULL , 30), +( 'caom2', 'caom2.ObsCoreEnumField', 'table', 'pre-computed aggregate (group by) materialised view built from enumerated types in ObsCore-1.1 model', NULL, 31), +( 'caom2', 'caom2.distinct_proposal_id', 'table', 'pre-computed materialised view of distinct caom2.Observation.proposal_id values', NULL , 32), +( 'caom2', 'caom2.distinct_proposal_pi', 'table', 'pre-computed materialised view of distinct caom2.Observation.proposal_pi values', NULL , 33), +( 'caom2', 'caom2.distinct_proposal_title', 'table', 'pre-computed materialised view of distinct caom2.Observation.proposal_title values', NULL , 34) ; -- EnumField diff --git a/caom2-tap-server/src/main/resources/sql/caom2.tap_schema_content11.sql b/caom2-tap-server/src/main/resources/sql/caom2.tap_schema_content11.sql index 7f10ddd..007daac 100644 --- a/caom2-tap-server/src/main/resources/sql/caom2.tap_schema_content11.sql +++ b/caom2-tap-server/src/main/resources/sql/caom2.tap_schema_content11.sql @@ -17,13 +17,11 @@ target_table in (select table_name from tap_schema.tables11 where schema_name = -- delete columns from tables in the caom2 schema delete from tap_schema.columns11 where table_name in -(select table_name from tap_schema.tables11 where schema_name = 'caom2' -and table_type='table' or table_name='caom2.SIAv1') +(select table_name from tap_schema.tables11 where schema_name = 'caom2') ; -- delete tables in the caom2 schema delete from tap_schema.tables11 where schema_name = 'caom2' -and table_type='table' or table_name='caom2.SIAv1' ; -- delete the caom2 schema diff --git a/caom2-tap-server/src/test/java/ca/nrc/cadc/tap/impl/InitCaomTapSchemaContentTest.java b/caom2-tap-server/src/test/java/ca/nrc/cadc/tap/impl/InitCaomTapSchemaContentTest.java index 6c3a353..d766da2 100644 --- a/caom2-tap-server/src/test/java/ca/nrc/cadc/tap/impl/InitCaomTapSchemaContentTest.java +++ b/caom2-tap-server/src/test/java/ca/nrc/cadc/tap/impl/InitCaomTapSchemaContentTest.java @@ -68,19 +68,17 @@ package ca.nrc.cadc.tap.impl; import ca.nrc.cadc.util.Log4jInit; - import java.io.PrintWriter; import java.sql.Connection; import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; import java.util.List; +import javax.sql.DataSource; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.junit.Assert; import org.junit.Test; -import javax.sql.DataSource; - /** * * @author pdowler @@ -98,8 +96,8 @@ public class InitCaomTapSchemaContentTest { @Test public void testParseCreateDDL() { try { - InitCaomTapSchemaContent init = new InitCaomTapSchemaContent(new TestDataSource(), null, schema); - for (String fname : InitCaomTapSchemaContent.CREATE_SQL) { + InitCaomTapSchemaContent init = new InitCaomTapSchemaContent(new TestDataSource(), null, schema, true); + for (String fname : InitCaomTapSchemaContent.BASE_EXTRA_SQL) { log.info("process file: " + fname); List statements = init.parseDDL(fname, schema); Assert.assertNotNull(statements); @@ -150,8 +148,8 @@ public void testParseCreateDDL() { @Test public void testParseUpgradeDDL() { try { - InitCaomTapSchemaContent init = new InitCaomTapSchemaContent(new TestDataSource(), null, schema); - for (String fname : InitCaomTapSchemaContent.UPGRADE_SQL) { + InitCaomTapSchemaContent init = new InitCaomTapSchemaContent(new TestDataSource(), null, schema, true); + for (String fname : InitCaomTapSchemaContent.BASE_EXTRA_SQL) { log.info("process file: " + fname); List statements = init.parseDDL(fname, schema); Assert.assertNotNull(statements);