Merge pull request #125 from biocodellc/develop

updated files to master
biocodellc · Mar 22, 2021 · dfa3fca · dfa3fca
2 parents babd9cb + 8f88d7d
commit dfa3fca
Show file tree

Hide file tree

Showing 23 changed files with 77 additions and 47 deletions.
diff --git a/PluginVersions.xml b/PluginVersions.xml
@@ -1,10 +1,7 @@
 <PluginVersions>
-    <LatestVersion>3.0.13</LatestVersion>
-    <LatestVersionURL>https://github.com/biocodellc/biocode-lims/releases/download/3.0.13/BiocodePlugin_3_0_12.gplugin</LatestVersionURL>
-    <ReleaseNotes>==Biocode Plugin 3.0.12 - 14 May 2019==
-       * Fixes bug in fetching tissues for plates
-       * Fixes "Already connected" exception occuring in Geneious v9
-       * Changed behaviour when creating new extractions that use existing extractions ID's
-    </ReleaseNotes>
+    <LatestVersion>3.0.15</LatestVersion>
+    <LatestVersionURL>https://github.com/biocodellc/biocode-lims/releases/download/3.0.15/BiocodePlugin_3_0_15.gplugin</LatestVersionURL>
+    <ReleaseNotes>==Biocode Plugin 3.0.15 - -24 Feb 2021==
+        </ReleaseNotes>
     <ExtraInformation/>
 </PluginVersions>
diff --git a/build.gradle b/build.gradle
@@ -19,7 +19,7 @@ plugins {
 apply plugin: "java"
 apply plugin: "idea"
 
-version = '3.0.13'
+version = '3.0.15'
 
 defaultTasks 'createPlugin';
 

diff --git a/doc/release_notes.txt b/doc/release_notes.txt
@@ -1,4 +1,15 @@
-==Biocode Plugin 3.0.11 - 14 May 2019==
+==Biocode Plugin 3.0.15 - 24 February 2021 ==
+* Fixes issues with large tissue queries against GEOME failing
+
+==Biocode Plugin 3.0.14 - 01 October 2020 ==
+* Fixes issue with detecting duplicate column names when annotating with FIMS data
+
+==Biocode Plugin 3.0.13 - 01 September 2020 ==
+* Fixes bug in mysql escaping
+* Fix column names to be case insensitive 
+* Documentation updates
+
+==Biocode Plugin 3.0.12 - 14 May 2019==
 * Fixes bug in fetching tissues for plates
 * Fixes "Already connected" exception occuring in Geneious v9
 * Changed behaviour when creating new extractions that use existing extractions ID's

diff --git a/src/com/biomatters/plugins/biocode/BiocodePlugin.java b/src/com/biomatters/plugins/biocode/BiocodePlugin.java
@@ -47,7 +47,7 @@ public class BiocodePlugin extends GeneiousPlugin {
     }
 
     private static GeneiousActionOptions superBiocodeAction;
-    public static final String PLUGIN_VERSION = "3.0.13";
+    public static final String PLUGIN_VERSION = "3.0.15";
     public static final String SUPPORT_EMAIL = "[email protected]";
 
     public static GeneiousActionOptions getSuperBiocodeAction() {

diff --git a/src/com/biomatters/plugins/biocode/assembler/annotate/AnnotateUtilities.java b/src/com/biomatters/plugins/biocode/assembler/annotate/AnnotateUtilities.java
@@ -100,7 +100,8 @@ public static void annotateFimsData(AnnotatedPluginDocument[] annotatedDocuments
 
         for (AnnotatedPluginDocument annotatedDocument : docsAnnotated) {
             for (DocumentField field : annotatedDocument.getDisplayableFields()) {
-                if(field.getCode().startsWith(MySQLFimsConnection.FIELD_PREFIX) || Pattern.matches("\\d+", field.getCode())) {
+                if(field.getCode().startsWith(MySQLFimsConnection.FIELD_PREFIX) || Pattern.matches("\\d+", field.getCode())
+                    || field.getCode().contains("urn:") )    {
                     if(annotatedDocument.getFieldValue(field) != null) {
                         oldFields.add(field);
                     }

diff --git a/src/com/biomatters/plugins/biocode/labbench/BiocodeService.java b/src/com/biomatters/plugins/biocode/labbench/BiocodeService.java
@@ -1067,7 +1067,7 @@ private AnnotatedPluginDocument createAssemblyDocument(AssembledSequence seq, St
             if (consensus == null || seq.date == null) {
                 consensus = "";
             } else if (seq.progress == null || seq.progress.toLowerCase().contains("failed")) {
-                consensus = "";
+                //consensus = "";
                 description = "Sequencing failed for this well";
             }
             consensus = consensus.replace("-", "");
@@ -1954,4 +1954,4 @@ public Plate getPlateForName(String plateName) throws DatabaseServiceException {
             throw new DatabaseServiceException(e, "Failed to download plate " + plateName + ": " + e.getMessage(), false);
         }
     }
-}
+}
diff --git a/src/com/biomatters/plugins/biocode/labbench/fims/geome/geomeFIMSConnection.java b/src/com/biomatters/plugins/biocode/labbench/fims/geome/geomeFIMSConnection.java
@@ -6,10 +6,8 @@
 import com.biomatters.geneious.publicapi.documents.PluginDocument;
 import com.biomatters.geneious.publicapi.plugin.Options;
 import com.biomatters.geneious.publicapi.utilities.StringUtilities;
-import com.biomatters.plugins.biocode.labbench.ConnectionException;
-import com.biomatters.plugins.biocode.labbench.FimsSample;
-import com.biomatters.plugins.biocode.labbench.PasswordOptions;
-import com.biomatters.plugins.biocode.labbench.TissueDocument;
+import com.biomatters.plugins.biocode.BiocodeUtilities;
+import com.biomatters.plugins.biocode.labbench.*;
 import com.biomatters.plugins.biocode.labbench.fims.FIMSConnection;
 import com.biomatters.plugins.biocode.labbench.fims.FimsProject;
 import com.biomatters.plugins.biocode.labbench.fims.TableFimsSample;
@@ -389,7 +387,8 @@ private Map<String, Map<String, Object>> mapResults(String idField, List<Map<Str
     }
 
     @Override
-    protected List<FimsSample> _retrieveSamplesForTissueIds(List<String> tissueIds, RetrieveCallback callback) throws ConnectionException {
+    protected List<FimsSample> _retrieveSamplesForTissueIds(List<String> tissueIds, RetrieveCallback rc) throws ConnectionException {
+
         try {
 
             // Strip out empty tissue IDs -- they will make queries to Geome fail parser check
@@ -402,29 +401,42 @@ protected List<FimsSample> _retrieveSamplesForTissueIds(List<String> tissueIds,
             Object[] trimmedTissueIds = tissueIdsArrayList.toArray();
 
             // Build Tissue query
-            Query[] tissueQueries = new Query[trimmedTissueIds.length];
-            for (int i = 0; i < trimmedTissueIds.length; i++) {
-                    tissueQueries[i] = Query.Factory.createFieldQuery(getTissueSampleDocumentField(), Condition.EQUAL, trimmedTissueIds[i].toString());
-            }
-            Query tissueQuery = Query.Factory.createOrQuery(tissueQueries, Collections.emptyMap());
-            String tissueIDsToQuery = buildQuery(tissueQuery);
-            String queryString =  tissueIDsToQuery + " _select_:[Tissue,Sample,Event]";
+            List<FimsSample> allSamples = new ArrayList<FimsSample>();
 
-            Invocation.Builder searchRequest = client.getQueryTarget().path("records/Tissue/json")
-                    .queryParam("limit", 100000)
-                    .request();
+            // Here we loop queries to GEOME in chunks of 1000 records each 
+            int chunk = 1000; // chunk size to divide
+            for(int cnt=0;cnt<trimmedTissueIds.length;cnt+=chunk) {
+                Object[] trimmedTissueIdsChunk = Arrays.copyOfRange(trimmedTissueIds, cnt, Math.min(trimmedTissueIds.length, cnt + chunk));
 
-            Form formToPost = new Form()
-                    .param("query", queryString)
-                    .param("entity", "Tissue");
+                //System.out.println("Downloading " + cnt + " of " + trimmedTissueIds.length);
 
-            Response response = searchRequest.post(
-                    Entity.entity(formToPost, MediaType.APPLICATION_FORM_URLENCODED_TYPE));
+                Query[] tissueQueries = new Query[trimmedTissueIdsChunk.length];
+                for (int i = 0; i < trimmedTissueIdsChunk.length; i++) {
+                    tissueQueries[i] = Query.Factory.createFieldQuery(getTissueSampleDocumentField(), Condition.EQUAL, trimmedTissueIdsChunk[i].toString());
+                }
+                Query tissueQuery = Query.Factory.createOrQuery(tissueQueries, Collections.emptyMap());
+                String tissueIDsToQuery = buildQuery(tissueQuery);
+                String queryString = tissueIDsToQuery + " _select_:[Tissue,Sample,Event]";
 
-            SearchResult result = geomeFIMSClient.getRestServiceResult(SearchResult.class, response);
+                Invocation.Builder searchRequest = client.getQueryTarget().path("records/Tissue/json")
+                        .queryParam("limit", chunk)
+                        .request();
+
+                Form formToPost = new Form()
+                        .param("query", queryString)
+                        .param("entity", "Tissue");
+
+                Response response = searchRequest.post(
+                        Entity.entity(formToPost, MediaType.APPLICATION_FORM_URLENCODED_TYPE));
+
+                SearchResult result = geomeFIMSClient.getRestServiceResult(SearchResult.class, response);
+
+                List<FimsSample> samples = transformQueryResults(tissueIds, result);
+
+                allSamples.addAll(samples);
+            }
 
-            List<FimsSample> samples = transformQueryResults(tissueIds, result);
-            return samples;
+            return allSamples;
 
         } catch (DatabaseServiceException e) {
             throw new ConnectionException(e);

diff --git a/src/com/biomatters/plugins/biocode/labbench/lims/FimsToLims.java b/src/com/biomatters/plugins/biocode/labbench/lims/FimsToLims.java
@@ -307,24 +307,31 @@ public void createFimsTable(final ProgressListener progress) throws ConnectionEx
             statement.executeUpdate(dropDefinitionTable);
 
             statement = lims.createStatement();
-            String dropFimsTable = "DROP TABLE IF EXISTS "+FIMS_VALUES_TABLE;
-            statement.executeUpdate(dropFimsTable);
+            //String dropFimsTable = "DROP TABLE IF EXISTS "+FIMS_VALUES_TABLE;
+            //statement.executeUpdate(dropFimsTable);
 
             final List<String> fieldsAndTypes = new ArrayList<String>();
             final List<String> fields = new ArrayList<String>();
             for(DocumentField f : fims.getSearchAttributes()) {
                 String colName = getSqlColName(f.getCode(), getLimsConnection().isLocal());
-                fieldsAndTypes.add(colName +" "+getColumnDefinition(f.getName().toLowerCase().contains("notes") ? null : f.getValueType()));
                 fields.add(colName);
+
+                // TODO: catch all mysql reserved words
+                if (colName.equalsIgnoreCase("order")) {
+                    colName = "ordr";
+                }
+                fieldsAndTypes.add(colName +" "+getColumnDefinition(f.getName().toLowerCase().contains("notes") ? null : f.getValueType()));
             }
             fieldsAndTypes.add("PRIMARY KEY ("+getSqlColName(fims.getTissueSampleDocumentField().getCode(), getLimsConnection().isLocal())+")");
 
-            String createValuesTable = "CREATE TABLE "+FIMS_VALUES_TABLE+"("+ StringUtilities.join(", ", fieldsAndTypes)+")";
-            System.out.println(createValuesTable);
-            statement.executeUpdate(createValuesTable);
+            // NOT dropping or creating table here... now setting to static.
+            //String createValuesTable = "CREATE TABLE "+FIMS_VALUES_TABLE+"("+ StringUtilities.join(", ", fieldsAndTypes)+")";
+            //System.out.println(createValuesTable);
+            //statement.executeUpdate(createValuesTable);
+
             final List<FimsSample> fimsSamples = new ArrayList<FimsSample>();
 
-            RetrieveCallback callback = new RetrieveCallback(listener){
+           /* RetrieveCallback callback = new RetrieveCallback(listener){
                 protected void _add(PluginDocument document, Map<String, Object> searchResultProperties) {
                     handle((FimsSample)document);
                 }
@@ -382,7 +389,7 @@ private void handle(FimsSample fimsSample) {
             if(listener.isCanceled()) {
                 return;
             }
-
+              */
             String createDefinitionTable;
             if(lims.isLocal()) {
                 createDefinitionTable = "CREATE TABLE " + FIMS_DEFINITION_TABLE + "(field VARCHAR(255) PRIMARY KEY,\n" +
@@ -391,12 +398,14 @@ private void handle(FimsSample fimsSample) {
             else {
                 createDefinitionTable = "CREATE TABLE " + FIMS_DEFINITION_TABLE + " (`field` varchar(255), `name` longtext, PRIMARY KEY  (`field`))";
             }
+
             statement.executeUpdate(createDefinitionTable);
             String fillDefinitionTable = "INSERT INTO " + FIMS_DEFINITION_TABLE + " (field, name) VALUES (?, ?)";
             PreparedStatement fillStatement = lims.createStatement(fillDefinitionTable);
             for(DocumentField f : fims.getSearchAttributes()) {
                 fillStatement.setString(1, getSqlColName(f.getCode(), getLimsConnection().isLocal()));
                 fillStatement.setString(2, f.getName());
+
                 fillStatement.executeUpdate();
             }
 
@@ -423,9 +432,9 @@ private void handle(FimsSample fimsSample) {
             ex.printStackTrace();
             Dialogs.showMessageDialog("There was an error copying your FIMS data into the LIMS: "+ex.getMessage());
 
-        } catch (DatabaseServiceException e) {
-            e.printStackTrace();
-            Dialogs.showMessageDialog("There was an error copying your FIMS data into the LIMS: "+e.getMessage());
+        //} catch (DatabaseServiceException e) {
+        //    e.printStackTrace();
+        //    Dialogs.showMessageDialog("There was an error copying your FIMS data into the LIMS: "+e.getMessage());
         } finally {
             listener.setProgress(1.0);
             BiocodeService.getInstance().unregisterCallback(listener);

diff --git a/wiki_images/96_to_384.png b/wiki_images/96_to_384.png
diff --git a/wiki_images/Annotate_with_fims.png b/wiki_images/Annotate_with_fims.png
diff --git a/wiki_images/Assembly.png b/wiki_images/Assembly.png
diff --git a/wiki_images/Assembly_report.png b/wiki_images/Assembly_report.png
diff --git a/wiki_images/Binning_parameters.png b/wiki_images/Binning_parameters.png
diff --git a/wiki_images/Bulk_add_traces.png b/wiki_images/Bulk_add_traces.png
diff --git a/wiki_images/Consenus_align.png b/wiki_images/Consenus_align.png
diff --git a/wiki_images/Download_traces.png b/wiki_images/Download_traces.png
diff --git a/wiki_images/Field_Login.png b/wiki_images/Field_Login.png
diff --git a/wiki_images/Genbank_submission.png b/wiki_images/Genbank_submission.png
diff --git a/wiki_images/Plate_viewer.png b/wiki_images/Plate_viewer.png
diff --git a/wiki_images/Search_box.png b/wiki_images/Search_box.png
diff --git a/wiki_images/Trimming.png b/wiki_images/Trimming.png
diff --git a/wiki_images/Verify_taxonomy.png b/wiki_images/Verify_taxonomy.png
diff --git a/wiki_images/cherry_picking.png b/wiki_images/cherry_picking.png