diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/SearchConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/SearchConfiguration.java index 24badabfdb0..728c524c9d3 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/SearchConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/SearchConfiguration.java @@ -23,6 +23,8 @@ import java.util.Collections; import java.util.List; +import static org.opencb.opencga.core.config.Configuration.reportUnusedField; + /** * Created by wasim on 09/11/16. */ @@ -36,20 +38,29 @@ public class SearchConfiguration { private String manager; private boolean active; private int timeout; + private int writeTimeout; private int insertBatchSize; private static final String DEFAULT_MODE = "cloud"; private static final boolean DEFAULT_ACTIVE = true; private static final int DEFAULT_TIMEOUT = 30000; + private static final int DEFAULT_WRITE_TIMEOUT = 120000; private static final int DEFAULT_INSERT_BATCH_SIZE = 10000; public SearchConfiguration() { - this(Collections.emptyList(), "", DEFAULT_MODE, "", "", "", DEFAULT_ACTIVE, DEFAULT_TIMEOUT, DEFAULT_INSERT_BATCH_SIZE); + this(Collections.emptyList(), "", DEFAULT_MODE, "", "", "", DEFAULT_ACTIVE, DEFAULT_TIMEOUT, DEFAULT_WRITE_TIMEOUT, + DEFAULT_INSERT_BATCH_SIZE); } + @Deprecated public SearchConfiguration(List hosts, String configSet, String mode, String user, String password, String manager, boolean active, int timeout, int insertBatchSize) { + this(hosts, configSet, mode, user, password, manager, active, timeout, DEFAULT_WRITE_TIMEOUT, insertBatchSize); + } + + public SearchConfiguration(List hosts, String configSet, String mode, String user, String password, String manager, + boolean active, int timeout, int writeTimeout, int insertBatchSize) { this.hosts = hosts; this.configSet = configSet; this.mode = mode; @@ -58,32 +69,36 @@ public SearchConfiguration(List hosts, String configSet, String mode, St this.manager = manager; this.active = active; this.timeout = timeout; + this.writeTimeout = writeTimeout; this.insertBatchSize = insertBatchSize; } @Override public String toString() { - return "SearchConfiguration{" + - "hosts=" + hosts + - ", configSet='" + configSet + '\'' + - ", mode='" + mode + '\'' + - ", user='" + user + '\'' + - ", password='" + password + '\'' + - ", manager='" + manager + '\'' + - ", active=" + active + - ", timeout=" + timeout + - ", insertBatchSize=" + insertBatchSize + - '}'; + final StringBuilder sb = new StringBuilder("SearchConfiguration{"); + sb.append("hosts=").append(hosts); + sb.append(", configSet='").append(configSet).append('\''); + sb.append(", mode='").append(mode).append('\''); + sb.append(", user='").append(user).append('\''); + sb.append(", password='").append(password).append('\''); + sb.append(", manager='").append(manager).append('\''); + sb.append(", active=").append(active); + sb.append(", timeout=").append(timeout); + sb.append(", writeTimeout=").append(writeTimeout); + sb.append(", insertBatchSize=").append(insertBatchSize); + sb.append('}'); + return sb.toString(); } @Deprecated public String getHost() { - return String.join(",", getHosts()); + return null; } @Deprecated public SearchConfiguration setHost(String host) { - return setHosts(StringUtils.isEmpty(host) ? Collections.emptyList() : Arrays.asList(host.split(","))); + reportUnusedField("search.configuration#host", host); + return this; } public List getHosts() { @@ -99,8 +114,9 @@ public String getConfigSet() { return configSet; } - public void setConfigSet(String configSet) { + public SearchConfiguration setConfigSet(String configSet) { this.configSet = configSet; + return this; } public String getMode() { @@ -157,17 +173,25 @@ public SearchConfiguration setTimeout(int timeout) { return this; } + public int getWriteTimeout() { + return writeTimeout; + } + + public SearchConfiguration setWriteTimeout(int writeTimeout) { + this.writeTimeout = writeTimeout; + return this; + } + @Deprecated public int getRows() { - return insertBatchSize; + return 0; } @Deprecated public SearchConfiguration setRows(int rows) { - this.insertBatchSize = rows; + reportUnusedField("search.configuration#rows", rows); return this; } - public int getInsertBatchSize() { return insertBatchSize; } diff --git a/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/executors/VariantCommandExecutor.java b/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/executors/VariantCommandExecutor.java index e564ae55f5c..b49538ab9d6 100644 --- a/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/executors/VariantCommandExecutor.java +++ b/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/executors/VariantCommandExecutor.java @@ -36,28 +36,28 @@ import org.opencb.commons.utils.FileUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.common.UriUtils; +import org.opencb.opencga.core.config.storage.StorageEngineConfiguration; +import org.opencb.opencga.core.models.common.mixins.GenericRecordAvroJsonMixin; import org.opencb.opencga.core.models.operations.variant.VariantAggregateFamilyParams; import org.opencb.opencga.core.models.operations.variant.VariantAggregateParams; import org.opencb.opencga.storage.app.cli.CommandExecutor; import org.opencb.opencga.storage.app.cli.GeneralCliOptions; import org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions; import org.opencb.opencga.storage.core.StorageEngineFactory; -import org.opencb.opencga.core.config.storage.StorageEngineConfiguration; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.models.ProjectMetadata; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.VariantStoragePipeline; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; -import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; import org.opencb.opencga.storage.core.variant.annotation.DefaultVariantAnnotationManager; import org.opencb.opencga.storage.core.variant.annotation.VariantAnnotationManager; import org.opencb.opencga.storage.core.variant.annotation.VariantAnnotatorException; import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory; -import org.opencb.opencga.core.models.common.mixins.GenericRecordAvroJsonMixin; -import org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager; +import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; import org.opencb.opencga.storage.core.variant.search.solr.SolrVariantDBIterator; +import org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager; import org.opencb.opencga.storage.core.variant.stats.DefaultVariantStatisticsManager; import java.io.*; @@ -68,8 +68,8 @@ import java.util.*; import java.util.function.Function; -import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.AggregateFamilyCommandOptions.AGGREGATE_FAMILY_COMMAND; import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.AggregateCommandOptions.AGGREGATE_COMMAND; +import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.AggregateFamilyCommandOptions.AGGREGATE_FAMILY_COMMAND; import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.GenericAnnotationDeleteCommandOptions.ANNOTATION_DELETE_COMMAND; import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.GenericAnnotationMetadataCommandOptions.ANNOTATION_METADATA_COMMAND; import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.GenericAnnotationQueryCommandOptions.ANNOTATION_QUERY_COMMAND; @@ -732,7 +732,7 @@ private void search() throws Exception { String solrUrl = (searchOptions.solrUrl == null ? "http://localhost:8983/solr/" : searchOptions.solrUrl); String dbName = (searchOptions.dbName == null ? "variants" : searchOptions.dbName); - variantStorageEngine.getConfiguration().getSearch().setHost(solrUrl); + variantStorageEngine.getConfiguration().getSearch().setHosts(Collections.singletonList(solrUrl)); // VariantSearchManager variantSearchManager = new VariantSearchManager(solrUrl, dbName); // VariantSearchManager variantSearchManager = new VariantSearchManager(variantStorageEngine.getStudyConfigurationManager(), diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java index 77327d9d76a..44cb8a12635 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java @@ -34,7 +34,6 @@ import org.opencb.opencga.core.models.operations.variant.VariantAggregateFamilyParams; import org.opencb.opencga.core.models.operations.variant.VariantAggregateParams; import org.opencb.opencga.core.models.variant.VariantSetupParams; -import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.StorageEngine; import org.opencb.opencga.storage.core.StoragePipelineResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; @@ -60,6 +59,7 @@ import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory.VariantOutputFormat; import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; import org.opencb.opencga.storage.core.variant.query.VariantQueryParser; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; import org.opencb.opencga.storage.core.variant.query.executors.*; import org.opencb.opencga.storage.core.variant.score.VariantScoreFormatDescriptor; @@ -793,8 +793,9 @@ protected void searchIndexLoadedFiles(List inputFiles, ObjectMap options) t } protected SolrInputDocumentDataWriter newVariantSearchDataWriter(String collection) throws StorageEngineException { + logger.info("Using SolrClient with a write timeout of {} ms", configuration.getSearch().getWriteTimeout()); return new SolrInputDocumentDataWriter(collection, - getVariantSearchManager().getSolrClient(), + getVariantSearchManager().getSolrManager().newSolrClient(configuration.getSearch().getWriteTimeout()), true, getVariantSearchManager().getInsertBatchSize()); } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrInputDocumentDataWriter.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrInputDocumentDataWriter.java index 5673652e052..6dc3b605f36 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrInputDocumentDataWriter.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrInputDocumentDataWriter.java @@ -9,12 +9,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.List; public class SolrInputDocumentDataWriter implements DataWriter { private final String collection; private final SolrClient solrClient; + private boolean closeSolrClient; private final int insertBatchSize; private int serverBufferSize = 0; private int insertedDocuments = 0; @@ -23,8 +25,13 @@ public class SolrInputDocumentDataWriter implements DataWriter batch) throws Exception { UpdateResponse response = solrClient.add(collection, batch); addTimeMs += response.getElapsedTime(); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/VariantSearchManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/VariantSearchManager.java index 374f5f9ffd6..6aaffcbec55 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/VariantSearchManager.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/VariantSearchManager.java @@ -693,11 +693,6 @@ public SolrClient getSolrClient() { return solrManager.getSolrClient(); } - public VariantSearchManager setSolrClient(SolrClient solrClient) { - this.solrManager.setSolrClient(solrClient); - return this; - } - public SolrQueryParser getSolrQueryParser() { return solrQueryParser; } diff --git a/opencga-storage/opencga-storage-core/src/main/resources/storage-configuration.yml b/opencga-storage/opencga-storage-core/src/main/resources/storage-configuration.yml index b9970d18eaf..faaba26d5fc 100644 --- a/opencga-storage/opencga-storage-core/src/main/resources/storage-configuration.yml +++ b/opencga-storage/opencga-storage-core/src/main/resources/storage-configuration.yml @@ -30,7 +30,8 @@ search: - ${OPENCGA.STORAGE.SEARCH.HOST} configSet: "${OPENCGA.STORAGE.SEARCH.CONFIG_SET}" mode: "cloud" - timeout: ${OPENCGA.STORAGE.SEARCH.TIMEOUT} + timeout: ${OPENCGA.STORAGE.SEARCH.TIMEOUT} # Timeout (in milliseconds) for read operations + writeTimeout: ${OPENCGA.STORAGE.SEARCH.WRITE.TIMEOUT} # Timeout (in milliseconds) for write operations insertBatchSize: 5000 ## Clinical database for indexing the pathogenic variants reported. diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageSearchIntersectTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageSearchIntersectTest.java index 28b7d0921ab..f50cabee188 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageSearchIntersectTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageSearchIntersectTest.java @@ -31,6 +31,7 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.commons.datastore.solr.SolrManager; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.StoragePipelineResult; @@ -92,8 +93,8 @@ public void before() throws Exception { // new Exception().printStackTrace(); // return invocation.callRealMethod(); // }).when(solrClient).query(anyString(), any()); + // Configure variant storage engine by using the current solrClient solr.configure(this.variantStorageEngine); - variantStorageEngine.getVariantSearchManager().setSolrClient(solrClient); for (VariantQueryExecutor variantQueryExecutor : variantStorageEngine.getVariantQueryExecutors()) { if (variantQueryExecutor instanceof SearchIndexVariantQueryExecutor) { this.variantQueryExecutor = ((SearchIndexVariantQueryExecutor) variantQueryExecutor); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java index dc5a019baa1..8a4469fff96 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java @@ -116,9 +116,8 @@ public VariantSearchManager configure(VariantStorageEngine variantStorageEngine) variantStorageEngine.getConfiguration().getSearch().setMode("core"); variantStorageEngine.getConfiguration().getSearch().setActive(true); VariantSearchManager variantSearchManager = variantStorageEngine.getVariantSearchManager(); - variantSearchManager.setSolrManager(new SolrManager(solrClient, "localhost", "core", - variantStorageEngine.getConfiguration().getSearch().getTimeout())); - variantSearchManager.setSolrClient(solrClient); + // Configure variant search manger by using the current solrClient + variantSearchManager.setSolrManager(new SolrManager(solrClient, "localhost", "core")); return variantSearchManager; } diff --git a/pom.xml b/pom.xml index e073746344d..f53b97b44d3 100644 --- a/pom.xml +++ b/pom.xml @@ -1397,6 +1397,7 @@ opencga-variant-configset-${project.parent.version} 30000 + 120000 http://localhost:8983/solr/ "" 30000