diff --git a/backend/src/main/java/uk/ac/ebi/spot/ols/Ols4Backend.java b/backend/src/main/java/uk/ac/ebi/spot/ols/Ols4Backend.java index 76608a943..76811ec70 100644 --- a/backend/src/main/java/uk/ac/ebi/spot/ols/Ols4Backend.java +++ b/backend/src/main/java/uk/ac/ebi/spot/ols/Ols4Backend.java @@ -7,6 +7,11 @@ import org.springframework.web.servlet.config.annotation.CorsRegistry; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; +/** + * TODO: We should consider caching responses that are static (such as statistics and ontologies loaded + * for the ontologies in the ontologies tab) on startup of the OLS backend. + * + */ @SpringBootApplication public class Ols4Backend { diff --git a/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v1/V1SearchController.java b/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v1/V1SearchController.java index 10cb89a2a..2102651f5 100644 --- a/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v1/V1SearchController.java +++ b/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v1/V1SearchController.java @@ -81,8 +81,6 @@ public void search( @RequestParam(value = "lang", defaultValue = "en") String lang, HttpServletResponse response ) throws IOException, SolrServerException { - System.out.println("fieldList 1 = " + fieldList); - System.out.println("type = " + types); final SolrQuery solrQuery = new SolrQuery(); // 1 @@ -208,15 +206,11 @@ public void search( * Fix: End */ solrQuery.add("wt", format); - System.out.println("fieldList 2 = " + fieldList); - - System.out.println("solrQuery=" + solrQuery.jsonStr()); QueryResponse qr = solrClient.dispatchSearch(solrQuery, "ols4_entities"); List docs = new ArrayList<>(); for(SolrDocument res : qr.getResults()) { - System.out.println("res = " + res.toString()); String _json = (String)res.get("_json"); if(_json == null) { throw new RuntimeException("_json was null"); diff --git a/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v1/V1SelectController.java b/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v1/V1SelectController.java index 38a4a34e7..bdd12ad47 100644 --- a/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v1/V1SelectController.java +++ b/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v1/V1SelectController.java @@ -6,12 +6,12 @@ import io.swagger.v3.oas.annotations.tags.Tag; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; -import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; @@ -42,6 +42,8 @@ public class V1SelectController { @Autowired private OlsSolrClient solrClient; + private static final Logger logger = LoggerFactory.getLogger(V1SelectController.class); + @RequestMapping(path = "/api/select", produces = {MediaType.APPLICATION_JSON_VALUE}, method = RequestMethod.GET) public void select( @RequestParam("q") String query, @@ -118,7 +120,7 @@ public void select( solrQuery.addHighlightField("whitespace_edge_synonym"); solrQuery.addHighlightField("synonym"); - System.out.println("select: " + solrQuery.toQueryString()); + logger.debug("select () ", solrQuery.toQueryString()); QueryResponse qr = solrClient.dispatchSearch(solrQuery, "ols4_entities"); diff --git a/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v2/V2OntologyController.java b/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v2/V2OntologyController.java index 6a9c8501e..4a57a67c6 100644 --- a/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v2/V2OntologyController.java +++ b/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v2/V2OntologyController.java @@ -51,6 +51,14 @@ public HttpEntity> getOntologies( @RequestParam(value = "includeObsoleteEntities", required = false, defaultValue = "false") boolean includeObsoleteEntities, @RequestParam Map> searchProperties ) throws ResourceNotFoundException, IOException { + logger.trace("Pageable: {}", pageable); + logger.trace("lang: {}", lang); + logger.trace("search: {}", search); + logger.trace("searchFields: {}", searchFields); + logger.trace("boostFields: {}", boostFields); + logger.trace("exactMatch: {}", exactMatch); + logger.trace("includeObsoleteEntities: {}", includeObsoleteEntities); + logger.trace("searchProperties: {}", searchProperties); Map> properties = new HashMap<>(); if(!includeObsoleteEntities) diff --git a/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v2/V2StatisticsController.java b/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v2/V2StatisticsController.java index 0af7b2460..dc89c0b3a 100644 --- a/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v2/V2StatisticsController.java +++ b/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v2/V2StatisticsController.java @@ -3,6 +3,8 @@ import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.response.FacetField; import org.apache.solr.client.solrj.response.QueryResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.rest.webmvc.ResourceNotFoundException; import org.springframework.hateoas.MediaTypes; @@ -27,36 +29,46 @@ public class V2StatisticsController { @Autowired OlsSolrClient solrClient; + private static ResponseEntity statisticsResponse = null ; + private static final Logger logger = LoggerFactory.getLogger(V2StatisticsController.class); + @RequestMapping(produces = {MediaType.APPLICATION_JSON_VALUE, MediaTypes.HAL_JSON_VALUE}, method = RequestMethod.GET) public HttpEntity getStatistics() throws ResourceNotFoundException, IOException { - Map coreStatus = solrClient.getCoreStatus(); - Map indexStatus = (Map) coreStatus.get("index"); - String lastModified = (String) indexStatus.get("lastModified"); + if (statisticsResponse == null) { + logger.debug("Getting statistics from Solr"); + Map coreStatus = solrClient.getCoreStatus(); + Map indexStatus = (Map) coreStatus.get("index"); + String lastModified = (String) indexStatus.get("lastModified"); - SolrQuery query = new SolrQuery(); + SolrQuery query = new SolrQuery(); - query.setQuery("*:*"); - query.setFacet(true); - query.addFacetField("type"); - query.setRows(0); + query.setQuery("*:*"); + query.setFacet(true); + query.addFacetField("type"); + query.setRows(0); - QueryResponse qr = solrClient.runSolrQuery(query, null); + QueryResponse qr = solrClient.runSolrQuery(query, null); - Map counts = new HashMap<>(); + Map counts = new HashMap<>(); - for(FacetField.Count count : qr.getFacetField("type").getValues()) { - counts.put(count.getName(), (int)count.getCount()); - } + for (FacetField.Count count : qr.getFacetField("type").getValues()) { + counts.put(count.getName(), (int) count.getCount()); + } - V2Statistics stats = new V2Statistics(); - stats.lastModified = lastModified; - stats.numberOfOntologies = counts.containsKey("ontology") ? counts.get("ontology") : 0; - stats.numberOfClasses = counts.containsKey("class") ? counts.get("class") : 0; - stats.numberOfIndividuals = counts.containsKey("individual") ? counts.get("individual") : 0; - stats.numberOfProperties = counts.containsKey("property") ? counts.get("property") : 0; - return new ResponseEntity<>( stats, HttpStatus.OK); - } + V2Statistics statistics = new V2Statistics(); + statistics.lastModified = lastModified; + statistics.numberOfOntologies = counts.containsKey("ontology") ? counts.get("ontology") : 0; + statistics.numberOfClasses = counts.containsKey("class") ? counts.get("class") : 0; + statistics.numberOfIndividuals = counts.containsKey("individual") ? counts.get("individual") : 0; + statistics.numberOfProperties = counts.containsKey("property") ? counts.get("property") : 0; + statisticsResponse = new ResponseEntity<>(statistics, HttpStatus.OK); + + } else { + logger.debug("Getting statistics from cache"); + } + return statisticsResponse; + } } diff --git a/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v2/cache/V2OntologyCacheController.java b/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v2/cache/V2OntologyCacheController.java new file mode 100644 index 000000000..27bfd928e --- /dev/null +++ b/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v2/cache/V2OntologyCacheController.java @@ -0,0 +1,72 @@ +package uk.ac.ebi.spot.ols.controller.api.v2.cache; + +import com.google.gson.Gson; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; +import org.springframework.data.rest.webmvc.ResourceNotFoundException; +import org.springframework.data.web.PageableDefault; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import uk.ac.ebi.spot.ols.controller.api.v2.helpers.DynamicQueryHelper; +import uk.ac.ebi.spot.ols.controller.api.v2.responses.V2PagedAndFacetedResponse; +import uk.ac.ebi.spot.ols.model.v2.V2Entity; +import uk.ac.ebi.spot.ols.repository.v2.V2OntologyRepository; + +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * This controller caches all ontologies for use in the ontologies tab. + * This is done to address OLS slowness as backend, solr and neo4j pods all located on different nodes. See + * https://overcast.blog/minimizing-inter-node-communication-in-kubernetes-dc53a8e28212. + * + */ +@Controller +@RequestMapping("/api/v2/cache/ontologies") +public class V2OntologyCacheController { + + private Gson gson = new Gson(); + + @Autowired + V2OntologyRepository ontologyRepository; + + private static ResponseEntity allOntologiesResponse = null; + + private static final Logger logger = LoggerFactory.getLogger(V2OntologyCacheController.class); + + @RequestMapping(path = "", produces = {MediaType.APPLICATION_JSON_VALUE }, method = RequestMethod.GET) + public HttpEntity> getAllOntologies( + ) throws ResourceNotFoundException, IOException { + + if (allOntologiesResponse == null) { + logger.debug("getAllOntologies from Solr"); + Map> properties = new HashMap<>(); + properties.put("isObsolete", List.of("false")); + + Pageable pageable = PageRequest.of(0, 1000, Sort.by("ontologyId")); + allOntologiesResponse = new ResponseEntity<>( + new V2PagedAndFacetedResponse<>( + ontologyRepository.find(pageable, "en", null, null, null, false, DynamicQueryHelper.filterProperties(properties)) + ), + HttpStatus.OK); + logger.trace("allOntologiesResponse = {}", allOntologiesResponse); + } else { + logger.debug("getAllOntologies from cache"); + } + return allOntologiesResponse; + } +} diff --git a/backend/src/main/java/uk/ac/ebi/spot/ols/repository/neo4j/OlsNeo4jClient.java b/backend/src/main/java/uk/ac/ebi/spot/ols/repository/neo4j/OlsNeo4jClient.java index 67e8b5681..7969529ba 100644 --- a/backend/src/main/java/uk/ac/ebi/spot/ols/repository/neo4j/OlsNeo4jClient.java +++ b/backend/src/main/java/uk/ac/ebi/spot/ols/repository/neo4j/OlsNeo4jClient.java @@ -1,11 +1,14 @@ package uk.ac.ebi.spot.ols.repository.neo4j; import com.google.gson.JsonElement; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Component; +import uk.ac.ebi.spot.ols.repository.solr.OlsSolrClient; import uk.ac.ebi.spot.ols.service.Neo4jClient; import java.util.List; @@ -19,6 +22,8 @@ public class OlsNeo4jClient { @Autowired Neo4jClient neo4jClient; + private static final Logger logger = LoggerFactory.getLogger(OlsNeo4jClient.class); + public Page getAll(String type, Map properties, Pageable pageable) { String query = "MATCH (a:" + type + ")"; @@ -71,40 +76,40 @@ private String makeEdgePropsClause(Map edgeProps) { public Page traverseOutgoingEdges(String type, String id, List edgeIRIs, Map edgeProps, Pageable pageable) { - String edge = makeEdgesList(edgeIRIs, edgeProps); + String edge = makeEdgesList(edgeIRIs, edgeProps); - // TODO fix injection + // TODO fix injection - String query = - "MATCH (a:" + type+ ")-[edge:" + edge + "]->(b) " - + "WHERE a.id = $id " + makeEdgePropsClause(edgeProps) - + "RETURN distinct b"; + String query = + "MATCH (a:" + type+ ")-[edge:" + edge + "]->(b) " + + "WHERE a.id = $id " + makeEdgePropsClause(edgeProps) + + "RETURN distinct b"; - String countQuery = - "MATCH (a:" + type + ")-[edge:" + edge + "]->(b) " - + "WHERE a.id = $id " + makeEdgePropsClause(edgeProps) - + "RETURN count(distinct b)"; + String countQuery = + "MATCH (a:" + type + ")-[edge:" + edge + "]->(b) " + + "WHERE a.id = $id " + makeEdgePropsClause(edgeProps) + + "RETURN count(distinct b)"; - System.out.println(query); + logger.debug("Query = {}", query); return neo4jClient.queryPaginated(query, "b", countQuery, parameters("type", type, "id", id), pageable); } public Page traverseIncomingEdges(String type, String id, List edgeIRIs, Map edgeProps, Pageable pageable) { - String edge = makeEdgesList(edgeIRIs, Map.of()); + String edge = makeEdgesList(edgeIRIs, Map.of()); - String query = - "MATCH (a:" + type + ")<-[edge:" + edge + "]-(b) " - + "WHERE a.id = $id " - + "RETURN distinct b"; + String query = + "MATCH (a:" + type + ")<-[edge:" + edge + "]-(b) " + + "WHERE a.id = $id " + + "RETURN distinct b"; - String countQuery = - "MATCH (a:" + type + ")<-[edge:" + edge + "]-(b) " - + "WHERE a.id = $id " - + "RETURN count(distinct b)"; + String countQuery = + "MATCH (a:" + type + ")<-[edge:" + edge + "]-(b) " + + "WHERE a.id = $id " + + "RETURN count(distinct b)"; - return neo4jClient.queryPaginated(query, "b", countQuery, parameters("type", type, "id", id), pageable); + return neo4jClient.queryPaginated(query, "b", countQuery, parameters("type", type, "id", id), pageable); } public Page recursivelyTraverseOutgoingEdges(String type, String id, List edgeIRIs, Map edgeProps, Pageable pageable) { @@ -128,21 +133,21 @@ public Page recursivelyTraverseOutgoingEdges(String type, String id public Page recursivelyTraverseIncomingEdges(String type, String id, List edgeIRIs, Map edgeProps, Pageable pageable) { - String edge = makeEdgesList(edgeIRIs, Map.of()); + String edge = makeEdgesList(edgeIRIs, Map.of()); - String query = - "MATCH (a:" + type + ") WHERE a.id = $id " - + "WITH a " - + "OPTIONAL MATCH (a)<-[edge:" + edge + " *]-(descendant) " - + "RETURN DISTINCT descendant AS c"; + String query = + "MATCH (a:" + type + ") WHERE a.id = $id " + + "WITH a " + + "OPTIONAL MATCH (a)<-[edge:" + edge + " *]-(descendant) " + + "RETURN DISTINCT descendant AS c"; - String countQuery = - "MATCH (a:" + type + ") WHERE a.id = $id " - + "WITH a " - + "OPTIONAL MATCH (a)<-[edge:" + edge + " *]-(descendant) " - + "RETURN count(DISTINCT descendant)"; + String countQuery = + "MATCH (a:" + type + ") WHERE a.id = $id " + + "WITH a " + + "OPTIONAL MATCH (a)<-[edge:" + edge + " *]-(descendant) " + + "RETURN count(DISTINCT descendant)"; - return neo4jClient.queryPaginated(query, "c", countQuery, parameters("id", id), pageable); + return neo4jClient.queryPaginated(query, "c", countQuery, parameters("id", id), pageable); } diff --git a/backend/src/main/java/uk/ac/ebi/spot/ols/service/Neo4jClient.java b/backend/src/main/java/uk/ac/ebi/spot/ols/service/Neo4jClient.java index cfbab9afd..97cd34934 100644 --- a/backend/src/main/java/uk/ac/ebi/spot/ols/service/Neo4jClient.java +++ b/backend/src/main/java/uk/ac/ebi/spot/ols/service/Neo4jClient.java @@ -12,12 +12,15 @@ import org.neo4j.driver.*; import org.neo4j.driver.Record; import org.neo4j.driver.exceptions.NoSuchRecordException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort; import org.springframework.stereotype.Component; import org.springframework.data.rest.webmvc.ResourceNotFoundException; +import uk.ac.ebi.spot.ols.repository.neo4j.OlsNeo4jClient; import javax.validation.constraints.NotNull; @@ -29,7 +32,7 @@ public class Neo4jClient { @org.springframework.beans.factory.annotation.Value("${ols.neo4j.host:bolt://localhost:7687}") private String host; - + private static final Logger logger = LoggerFactory.getLogger(Neo4jClient.class); private Gson gson = new Gson(); @@ -106,8 +109,8 @@ public Page queryPaginated(String query, String resVar, String coun queryToRun = query; } - System.out.println(queryToRun); - System.out.println(gson.toJson(parameters.asMap())); + logger.trace("queryToRun = {}", queryToRun); + logger.info(" gson.toJson(parameters.asMap()) = {}", gson.toJson(parameters.asMap())); Stopwatch timer = Stopwatch.createStarted(); Result result = session.run( @@ -115,11 +118,11 @@ public Page queryPaginated(String query, String resVar, String coun parameters ); - System.out.println("Neo4j run paginated query: " + timer.stop()); + logger.info("Neo4j run paginated query duration = {}", timer.stop()); Stopwatch timer2 = Stopwatch.createStarted(); Result countResult = session.run(countQuery, parameters); - System.out.println("Neo4j run paginated count: " + timer2.stop()); + logger.info("Neo4j run paginated count duration = {}", timer2.stop()); Record countRecord = countResult.single(); int count = countRecord.get(0).asInt(); @@ -142,11 +145,11 @@ public JsonElement queryOne(String query, String resVar, Value parameters) { Session session = getSession(); - System.out.println(query); + logger.debug("query = {}", query); Stopwatch timer = Stopwatch.createStarted(); Result result = session.run(query, parameters); - System.out.println("Neo4j run query " + query + ": " + timer.stop()); + logger.info("Neo4j run query {} has duration {}", query, timer.stop()); Value v = null; diff --git a/backend/src/main/resources/logback.xml b/backend/src/main/resources/logback.xml index 2c1bbe530..caedf23aa 100644 --- a/backend/src/main/resources/logback.xml +++ b/backend/src/main/resources/logback.xml @@ -7,7 +7,7 @@ - + diff --git a/frontend/src/pages/ontologies/ontologiesSlice.ts b/frontend/src/pages/ontologies/ontologiesSlice.ts index e11ca825a..73c44e51c 100644 --- a/frontend/src/pages/ontologies/ontologiesSlice.ts +++ b/frontend/src/pages/ontologies/ontologiesSlice.ts @@ -252,20 +252,14 @@ export const getOntologies = createAsyncThunk( export const getAllOntologies = createAsyncThunk( "ontologies_all", async (_, { rejectWithValue }) => { - const path = `api/v2/ontologies?size=1`; + const allOntologiesPath = `api/v2/cache/ontologies`; try { - const response = await get(path); - const totalElements = response.totalElements; - - const allOntologiesPath = `api/v2/ontologies?size=${totalElements}`; const allOntologiesResponse = await get(allOntologiesPath); const data = allOntologiesResponse.elements.map((o: any) => new Ontology(o)); - //const data = (await getPaginated(path)).map((o) => new Ontology(o)); - return data; } catch (error: any) { - return rejectWithValue(`Error accessing: ${path}; ${error.message}`); + return rejectWithValue(`Error accessing: ${allOntologiesPath}; ${error.message}`); } } );