Merge pull request #153 from pdowler/master

argus: add option to add materialized views to tap_schema

argus/README.md

@@ -71,7 +71,16 @@ using the _TempStorageManager_, the base URL must include "/results" as the last
 (e.g. `https://example.net/argus/results`).
 
 ### argus.properties
-This file may be needed in the future but is not currently used.
+This file is currently optional and configures optional service features.
+```
+# enable materialized views (optional; default: false)
+org.opencadc.argus.enableMaterializedViews = true | false
+```
+
+The _enableMaterializedViews_ key causes `argus` to add descriptions of materialized views to the TAP_SCHEMA
+for use in queries. NOTE: enabling this feature does not currrently create or maintain the views so only enable
+this if you have out-of-band creation working. TODO: implement creation and periodic update of materialized
+views in `argus`.
 
 ### injecting VOTable resources into query results
 Some columns in the CAOM metadata have _ID_ values that are asssigned to the VOTable FIELD in the output.

argus/VERSION

@@ -1,6 +1,6 @@
 ## deployable containers have a semantic and build tag
 # semantic version tag: major.minor
 # build version tag: timestamp
-VER=1.0.8
+VER=1.0.9
 TAGS="${VER} ${VER}-$(date -u +"%Y%m%dT%H%M%S")"
 unset VER

argus/src/main/java/org/opencadc/argus/ArgusInitAction.java

@@ -3,7 +3,7 @@
 *******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
 **************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
 *
-*  (c) 2023.                            (c) 2023.
+*  (c) 2024.                            (c) 2024.
 *  Government of Canada                 Gouvernement du Canada
 *  National Research Council            Conseil national de recherches
 *  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
@@ -71,6 +71,8 @@
 import ca.nrc.cadc.rest.InitAction;
 import ca.nrc.cadc.tap.impl.InitCaomTapSchemaContent;
 import ca.nrc.cadc.tap.schema.InitDatabaseTS;
+import ca.nrc.cadc.util.MultiValuedProperties;
+import ca.nrc.cadc.util.PropertiesReader;
 import ca.nrc.cadc.uws.server.impl.InitDatabaseUWS;
 import javax.sql.DataSource;
 import org.apache.log4j.Logger;
@@ -84,7 +86,22 @@ public class ArgusInitAction extends InitAction {
 
     private static final Logger log = Logger.getLogger(ArgusInitAction.class);
 
+    private static final String CONFIG_ENABLE_MV = "org.opencadc.argus.enableMaterializedViews";
+
+    private final boolean enableMaterialisedViews;
+
     public ArgusInitAction() {
+        PropertiesReader pr = new PropertiesReader("argus.properties");
+        boolean enableMV = false;
+        try {
+            MultiValuedProperties mvp = pr.getAllProperties();
+
+            String str = mvp.getFirstPropertyValue(CONFIG_ENABLE_MV);
+            enableMV = "true".equals(str);
+        } catch (Exception ex) {
+            log.warn("failed to read otional config: " + ex);
+        }
+        this.enableMaterialisedViews = enableMV;
     }
 
     @Override
@@ -106,7 +123,7 @@ public void doInit() {
 
             // caom2 tap_schema content
             log.info("InitCaomTapSchemaContent: START");
-            InitCaomTapSchemaContent lsc = new InitCaomTapSchemaContent(tapadm, null, "tap_schema");
+            InitCaomTapSchemaContent lsc = new InitCaomTapSchemaContent(tapadm, null, "tap_schema", enableMaterialisedViews);
             lsc.doInit();
             log.info("InitCaomTapSchemaContent: OK");
         } catch (Exception ex) {

caom2-tap-server/build.gradle

@@ -17,7 +17,7 @@ sourceCompatibility = 1.8
 
 group = 'org.opencadc'
 
-version = '1.2.12'
+version = '1.2.13'
 
 description = 'OpenCADC CAOM2 TAP server library'
 def git_url = 'https://github.com/opencadc/caom2service'

caom2-tap-server/src/main/java/ca/nrc/cadc/tap/impl/InitCaomTapSchemaContent.java

@@ -3,7 +3,7 @@
 *******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
 **************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
 *
-*  (c) 2023.                            (c) 2023.
+*  (c) 2024.                            (c) 2024.
 *  Government of Canada                 Gouvernement du Canada
 *  National Research Council            Conseil national de recherches
 *  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
@@ -87,17 +87,18 @@ public class InitCaomTapSchemaContent extends InitDatabase {
     private static final Logger log = Logger.getLogger(InitCaomTapSchemaContent.class);
 
     public static final String MODEL_NAME = "caom-schema";
-    public static final String MODEL_VERSION = "1.2.12";
+    public static final String MODEL_VERSION = "1.2.13";
 
     // the SQL is tightly coupled to cadc-tap-schema table names (for TAP-1.1)
-    static String[] CREATE_SQL = new String[] {
+    static String[] BASE_SQL = new String[] {
         "caom2.tap_schema_content11.sql",
         "ivoa.tap_schema_content11.sql"
     };
 
     // upgrade is normally the same as create since SQL is idempotent
-    static String[] UPGRADE_SQL = new String[] {
+    static String[] BASE_EXTRA_SQL = new String[] {
         "caom2.tap_schema_content11.sql",
+        "caom2.tap_schema_content11-extra.sql",
         "ivoa.tap_schema_content11.sql"
     };
 
@@ -108,13 +109,18 @@ public class InitCaomTapSchemaContent extends InitDatabase {
      * @param dataSource connection with write permission to tap_schema tables
      * @param database database name (should be null if not needed in SQL)
      * @param schema schema name (usually tap_schema)
+     * @param extras add tap_schema content for extra tables (materialised views)
      */
-    public InitCaomTapSchemaContent(DataSource dataSource, String database, String schema) {
-        super(dataSource, database, schema, MODEL_NAME, MODEL_VERSION);
-        for (String s : CREATE_SQL) {
-            createSQL.add(s);
+    public InitCaomTapSchemaContent(DataSource dataSource, String database, String schema, boolean extras) {
+        // use MODELVERSION/extras so changing extras will cause a recreate
+        // eg 1.2.13/false <-> 1.2.13/true
+        super(dataSource, database, schema, MODEL_NAME, MODEL_VERSION + "/" + extras);
+        String[] src = BASE_SQL;
+        if (extras) {
+            src = BASE_EXTRA_SQL;
         }
-        for (String s : UPGRADE_SQL) {
+        for (String s : src) {
+            createSQL.add(s);
             upgradeSQL.add(s);
         }
     }

caom2-tap-server/src/main/resources/sql/caom2.tap_schema_content11-extra.sql

@@ -2,11 +2,11 @@
 
 insert into tap_schema.tables11 (schema_name,table_name,table_type,description,utype,table_index) values
 -- index start at 30
-( 'caom2', 'caom2.EnumField', 'view', 'pre-computed aggregate (group by) materialised view built from enumerated types in CAOM model', NULL , 30),
-( 'caom2', 'caom2.ObsCoreEnumField', 'view', 'pre-computed aggregate (group by) materialised view built from enumerated types in ObsCore-1.1 model', NULL, 31),
-( 'caom2', 'caom2.distinct_proposal_id', 'view', 'pre-computed materialised view of distinct caom2.Observation.proposal_id values', NULL , 32),
-( 'caom2', 'caom2.distinct_proposal_pi', 'view', 'pre-computed materialised view of distinct caom2.Observation.proposal_pi values', NULL , 33),
-( 'caom2', 'caom2.distinct_proposal_title', 'view', 'pre-computed materialised view of distinct caom2.Observation.proposal_title values', NULL , 34)
+( 'caom2', 'caom2.EnumField', 'table', 'pre-computed aggregate (group by) materialised view built from enumerated types in CAOM model', NULL , 30),
+( 'caom2', 'caom2.ObsCoreEnumField', 'table', 'pre-computed aggregate (group by) materialised view built from enumerated types in ObsCore-1.1 model', NULL, 31),
+( 'caom2', 'caom2.distinct_proposal_id', 'table', 'pre-computed materialised view of distinct caom2.Observation.proposal_id values', NULL , 32),
+( 'caom2', 'caom2.distinct_proposal_pi', 'table', 'pre-computed materialised view of distinct caom2.Observation.proposal_pi values', NULL , 33),
+( 'caom2', 'caom2.distinct_proposal_title', 'table', 'pre-computed materialised view of distinct caom2.Observation.proposal_title values', NULL , 34)
 ;
 
 -- EnumField

caom2-tap-server/src/main/resources/sql/caom2.tap_schema_content11.sql

@@ -17,13 +17,11 @@ target_table in (select table_name from tap_schema.tables11 where schema_name =
 
 -- delete columns from tables in the caom2 schema
 delete from tap_schema.columns11 where table_name in 
-(select table_name from tap_schema.tables11 where schema_name = 'caom2'
-and table_type='table' or table_name='caom2.SIAv1')
+(select table_name from tap_schema.tables11 where schema_name = 'caom2')
 ;
 
 -- delete tables in the caom2 schema
 delete from tap_schema.tables11 where schema_name = 'caom2'
-and table_type='table' or table_name='caom2.SIAv1'
 ;
 
 -- delete the caom2 schema

caom2-tap-server/src/test/java/ca/nrc/cadc/tap/impl/InitCaomTapSchemaContentTest.java

@@ -68,19 +68,17 @@
 package ca.nrc.cadc.tap.impl;
 
 import ca.nrc.cadc.util.Log4jInit;
-
 import java.io.PrintWriter;
 import java.sql.Connection;
 import java.sql.SQLException;
 import java.sql.SQLFeatureNotSupportedException;
 import java.util.List;
+import javax.sql.DataSource;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 import org.junit.Assert;
 import org.junit.Test;
 
-import javax.sql.DataSource;
-
 /**
  *
  * @author pdowler
@@ -98,8 +96,8 @@ public class InitCaomTapSchemaContentTest {
     @Test
     public void testParseCreateDDL() {
         try {
-            InitCaomTapSchemaContent init = new InitCaomTapSchemaContent(new TestDataSource(), null, schema);
-            for (String fname : InitCaomTapSchemaContent.CREATE_SQL) {
+            InitCaomTapSchemaContent init = new InitCaomTapSchemaContent(new TestDataSource(), null, schema, true);
+            for (String fname : InitCaomTapSchemaContent.BASE_EXTRA_SQL) {
                 log.info("process file: " + fname);
                 List<String> statements = init.parseDDL(fname, schema);
                 Assert.assertNotNull(statements);
@@ -150,8 +148,8 @@ public void testParseCreateDDL() {
     @Test
     public void testParseUpgradeDDL() {
         try {
-            InitCaomTapSchemaContent init = new InitCaomTapSchemaContent(new TestDataSource(), null, schema);
-            for (String fname : InitCaomTapSchemaContent.UPGRADE_SQL) {
+            InitCaomTapSchemaContent init = new InitCaomTapSchemaContent(new TestDataSource(), null, schema, true);
+            for (String fname : InitCaomTapSchemaContent.BASE_EXTRA_SQL) {
                 log.info("process file: " + fname);
                 List<String> statements = init.parseDDL(fname, schema);
                 Assert.assertNotNull(statements);