Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cached statistics and all ontologies for ontologies tab and cleaned u… #700

Open
wants to merge 2 commits into
base: dev
Choose a base branch
from
Open

Cached statistics and all ontologies for ontologies tab and cleaned u… #700

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
09d8251
Cached statistics and all ontologies for ontologies tab and cleaned u…
henrietteharmse Jul 13, 2024
47538f8
Sort ontologies for ontologies tab
henrietteharmse Jul 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions backend/src/main/java/uk/ac/ebi/spot/ols/Ols4Backend.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@
import org.springframework.web.servlet.config.annotation.CorsRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;

/**
* TODO: We should consider caching responses that are static (such as statistics and ontologies loaded
* for the ontologies in the ontologies tab) on startup of the OLS backend.
*
*/
@SpringBootApplication
public class Ols4Backend {

Expand Down
6 changes: 0 additions & 6 deletions backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v1/V1SearchController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,6 @@ public void search(
@RequestParam(value = "lang", defaultValue = "en") String lang,
HttpServletResponse response
) throws IOException, SolrServerException {
System.out.println("fieldList 1 = " + fieldList);
System.out.println("type = " + types);

final SolrQuery solrQuery = new SolrQuery(); // 1

Expand Down Expand Up @@ -208,15 +206,11 @@ public void search(
* Fix: End
*/
solrQuery.add("wt", format);
System.out.println("fieldList 2 = " + fieldList);

System.out.println("solrQuery=" + solrQuery.jsonStr());

QueryResponse qr = solrClient.dispatchSearch(solrQuery, "ols4_entities");

List<Object> docs = new ArrayList<>();
for(SolrDocument res : qr.getResults()) {
System.out.println("res = " + res.toString());
String _json = (String)res.get("_json");
if(_json == null) {
throw new RuntimeException("_json was null");
Expand Down
8 changes: 5 additions & 3 deletions backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v1/V1SelectController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@
import io.swagger.v3.oas.annotations.tags.Tag;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
Expand Down Expand Up @@ -42,6 +42,8 @@ public class V1SelectController {
@Autowired
private OlsSolrClient solrClient;

private static final Logger logger = LoggerFactory.getLogger(V1SelectController.class);

@RequestMapping(path = "/api/select", produces = {MediaType.APPLICATION_JSON_VALUE}, method = RequestMethod.GET)
public void select(
@RequestParam("q") String query,
Expand Down Expand Up @@ -118,7 +120,7 @@ public void select(
solrQuery.addHighlightField("whitespace_edge_synonym");
solrQuery.addHighlightField("synonym");

System.out.println("select: " + solrQuery.toQueryString());
logger.debug("select () ", solrQuery.toQueryString());

QueryResponse qr = solrClient.dispatchSearch(solrQuery, "ols4_entities");

Expand Down
8 changes: 8 additions & 0 deletions backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v2/V2OntologyController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,14 @@ public HttpEntity<V2PagedAndFacetedResponse<V2Entity>> getOntologies(
@RequestParam(value = "includeObsoleteEntities", required = false, defaultValue = "false") boolean includeObsoleteEntities,
@RequestParam Map<String, Collection<String>> searchProperties
) throws ResourceNotFoundException, IOException {
logger.trace("Pageable: {}", pageable);
logger.trace("lang: {}", lang);
logger.trace("search: {}", search);
logger.trace("searchFields: {}", searchFields);
logger.trace("boostFields: {}", boostFields);
logger.trace("exactMatch: {}", exactMatch);
logger.trace("includeObsoleteEntities: {}", includeObsoleteEntities);
logger.trace("searchProperties: {}", searchProperties);

Map<String,Collection<String>> properties = new HashMap<>();
if(!includeObsoleteEntities)
Expand Down
54 changes: 33 additions & 21 deletions backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v2/V2StatisticsController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.rest.webmvc.ResourceNotFoundException;
import org.springframework.hateoas.MediaTypes;
Expand All @@ -27,36 +29,46 @@ public class V2StatisticsController {
@Autowired
OlsSolrClient solrClient;

private static ResponseEntity statisticsResponse = null ;
private static final Logger logger = LoggerFactory.getLogger(V2StatisticsController.class);

@RequestMapping(produces = {MediaType.APPLICATION_JSON_VALUE, MediaTypes.HAL_JSON_VALUE}, method = RequestMethod.GET)
public HttpEntity<V2Statistics> getStatistics() throws ResourceNotFoundException, IOException {

Map<String,Object> coreStatus = solrClient.getCoreStatus();
Map<String,Object> indexStatus = (Map<String,Object>) coreStatus.get("index");
String lastModified = (String) indexStatus.get("lastModified");
if (statisticsResponse == null) {
logger.debug("Getting statistics from Solr");
Map<String, Object> coreStatus = solrClient.getCoreStatus();
Map<String, Object> indexStatus = (Map<String, Object>) coreStatus.get("index");
String lastModified = (String) indexStatus.get("lastModified");

SolrQuery query = new SolrQuery();
SolrQuery query = new SolrQuery();

query.setQuery("*:*");
query.setFacet(true);
query.addFacetField("type");
query.setRows(0);
query.setQuery("*:*");
query.setFacet(true);
query.addFacetField("type");
query.setRows(0);

QueryResponse qr = solrClient.runSolrQuery(query, null);
QueryResponse qr = solrClient.runSolrQuery(query, null);

Map<String,Integer> counts = new HashMap<>();
Map<String, Integer> counts = new HashMap<>();

for(FacetField.Count count : qr.getFacetField("type").getValues()) {
counts.put(count.getName(), (int)count.getCount());
}
for (FacetField.Count count : qr.getFacetField("type").getValues()) {
counts.put(count.getName(), (int) count.getCount());
}

V2Statistics stats = new V2Statistics();
stats.lastModified = lastModified;
stats.numberOfOntologies = counts.containsKey("ontology") ? counts.get("ontology") : 0;
stats.numberOfClasses = counts.containsKey("class") ? counts.get("class") : 0;
stats.numberOfIndividuals = counts.containsKey("individual") ? counts.get("individual") : 0;
stats.numberOfProperties = counts.containsKey("property") ? counts.get("property") : 0;

return new ResponseEntity<>( stats, HttpStatus.OK);
}
V2Statistics statistics = new V2Statistics();
statistics.lastModified = lastModified;
statistics.numberOfOntologies = counts.containsKey("ontology") ? counts.get("ontology") : 0;
statistics.numberOfClasses = counts.containsKey("class") ? counts.get("class") : 0;
statistics.numberOfIndividuals = counts.containsKey("individual") ? counts.get("individual") : 0;
statistics.numberOfProperties = counts.containsKey("property") ? counts.get("property") : 0;

statisticsResponse = new ResponseEntity<>(statistics, HttpStatus.OK);

} else {
logger.debug("Getting statistics from cache");
}
return statisticsResponse;
}
}
72 changes: 72 additions & 0 deletions ...d/src/main/java/uk/ac/ebi/spot/ols/controller/api/v2/cache/V2OntologyCacheController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
package uk.ac.ebi.spot.ols.controller.api.v2.cache;

import com.google.gson.Gson;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.data.rest.webmvc.ResourceNotFoundException;
import org.springframework.data.web.PageableDefault;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import uk.ac.ebi.spot.ols.controller.api.v2.helpers.DynamicQueryHelper;
import uk.ac.ebi.spot.ols.controller.api.v2.responses.V2PagedAndFacetedResponse;
import uk.ac.ebi.spot.ols.model.v2.V2Entity;
import uk.ac.ebi.spot.ols.repository.v2.V2OntologyRepository;

import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
* This controller caches all ontologies for use in the ontologies tab.
* This is done to address OLS slowness as backend, solr and neo4j pods all located on different nodes. See
* https://overcast.blog/minimizing-inter-node-communication-in-kubernetes-dc53a8e28212.
*
*/
@Controller
@RequestMapping("/api/v2/cache/ontologies")
public class V2OntologyCacheController {

private Gson gson = new Gson();

@Autowired
V2OntologyRepository ontologyRepository;

private static ResponseEntity allOntologiesResponse = null;

private static final Logger logger = LoggerFactory.getLogger(V2OntologyCacheController.class);

@RequestMapping(path = "", produces = {MediaType.APPLICATION_JSON_VALUE }, method = RequestMethod.GET)
public HttpEntity<V2PagedAndFacetedResponse<V2Entity>> getAllOntologies(
) throws ResourceNotFoundException, IOException {

if (allOntologiesResponse == null) {
logger.debug("getAllOntologies from Solr");
Map<String, Collection<String>> properties = new HashMap<>();
properties.put("isObsolete", List.of("false"));

Pageable pageable = PageRequest.of(0, 1000, Sort.by("ontologyId"));
allOntologiesResponse = new ResponseEntity<>(
new V2PagedAndFacetedResponse<>(
ontologyRepository.find(pageable, "en", null, null, null, false, DynamicQueryHelper.filterProperties(properties))
),
HttpStatus.OK);
logger.trace("allOntologiesResponse = {}", allOntologiesResponse);
} else {
logger.debug("getAllOntologies from cache");
}
return allOntologiesResponse;
}
}
71 changes: 38 additions & 33 deletions backend/src/main/java/uk/ac/ebi/spot/ols/repository/neo4j/OlsNeo4jClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
package uk.ac.ebi.spot.ols.repository.neo4j;

import com.google.gson.JsonElement;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Component;
import uk.ac.ebi.spot.ols.repository.solr.OlsSolrClient;
import uk.ac.ebi.spot.ols.service.Neo4jClient;

import java.util.List;
Expand All @@ -19,6 +22,8 @@ public class OlsNeo4jClient {
@Autowired
Neo4jClient neo4jClient;

private static final Logger logger = LoggerFactory.getLogger(OlsNeo4jClient.class);

public Page<JsonElement> getAll(String type, Map<String,String> properties, Pageable pageable) {

String query = "MATCH (a:" + type + ")";
Expand Down Expand Up @@ -71,40 +76,40 @@ private String makeEdgePropsClause(Map<String,String> edgeProps) {

public Page<JsonElement> traverseOutgoingEdges(String type, String id, List<String> edgeIRIs, Map<String,String> edgeProps, Pageable pageable) {

String edge = makeEdgesList(edgeIRIs, edgeProps);
String edge = makeEdgesList(edgeIRIs, edgeProps);

// TODO fix injection
// TODO fix injection

String query =
"MATCH (a:" + type+ ")-[edge:" + edge + "]->(b) "
+ "WHERE a.id = $id " + makeEdgePropsClause(edgeProps)
+ "RETURN distinct b";
String query =
"MATCH (a:" + type+ ")-[edge:" + edge + "]->(b) "
+ "WHERE a.id = $id " + makeEdgePropsClause(edgeProps)
+ "RETURN distinct b";

String countQuery =
"MATCH (a:" + type + ")-[edge:" + edge + "]->(b) "
+ "WHERE a.id = $id " + makeEdgePropsClause(edgeProps)
+ "RETURN count(distinct b)";
String countQuery =
"MATCH (a:" + type + ")-[edge:" + edge + "]->(b) "
+ "WHERE a.id = $id " + makeEdgePropsClause(edgeProps)
+ "RETURN count(distinct b)";

System.out.println(query);
logger.debug("Query = {}", query);

return neo4jClient.queryPaginated(query, "b", countQuery, parameters("type", type, "id", id), pageable);
}

public Page<JsonElement> traverseIncomingEdges(String type, String id, List<String> edgeIRIs, Map<String,String> edgeProps, Pageable pageable) {

String edge = makeEdgesList(edgeIRIs, Map.of());
String edge = makeEdgesList(edgeIRIs, Map.of());

String query =
"MATCH (a:" + type + ")<-[edge:" + edge + "]-(b) "
+ "WHERE a.id = $id "
+ "RETURN distinct b";
String query =
"MATCH (a:" + type + ")<-[edge:" + edge + "]-(b) "
+ "WHERE a.id = $id "
+ "RETURN distinct b";

String countQuery =
"MATCH (a:" + type + ")<-[edge:" + edge + "]-(b) "
+ "WHERE a.id = $id "
+ "RETURN count(distinct b)";
String countQuery =
"MATCH (a:" + type + ")<-[edge:" + edge + "]-(b) "
+ "WHERE a.id = $id "
+ "RETURN count(distinct b)";

return neo4jClient.queryPaginated(query, "b", countQuery, parameters("type", type, "id", id), pageable);
return neo4jClient.queryPaginated(query, "b", countQuery, parameters("type", type, "id", id), pageable);
}

public Page<JsonElement> recursivelyTraverseOutgoingEdges(String type, String id, List<String> edgeIRIs, Map<String,String> edgeProps, Pageable pageable) {
Expand All @@ -128,21 +133,21 @@ public Page<JsonElement> recursivelyTraverseOutgoingEdges(String type, String id

public Page<JsonElement> recursivelyTraverseIncomingEdges(String type, String id, List<String> edgeIRIs, Map<String,String> edgeProps, Pageable pageable) {

String edge = makeEdgesList(edgeIRIs, Map.of());
String edge = makeEdgesList(edgeIRIs, Map.of());

String query =
"MATCH (a:" + type + ") WHERE a.id = $id "
+ "WITH a "
+ "OPTIONAL MATCH (a)<-[edge:" + edge + " *]-(descendant) "
+ "RETURN DISTINCT descendant AS c";
String query =
"MATCH (a:" + type + ") WHERE a.id = $id "
+ "WITH a "
+ "OPTIONAL MATCH (a)<-[edge:" + edge + " *]-(descendant) "
+ "RETURN DISTINCT descendant AS c";

String countQuery =
"MATCH (a:" + type + ") WHERE a.id = $id "
+ "WITH a "
+ "OPTIONAL MATCH (a)<-[edge:" + edge + " *]-(descendant) "
+ "RETURN count(DISTINCT descendant)";
String countQuery =
"MATCH (a:" + type + ") WHERE a.id = $id "
+ "WITH a "
+ "OPTIONAL MATCH (a)<-[edge:" + edge + " *]-(descendant) "
+ "RETURN count(DISTINCT descendant)";

return neo4jClient.queryPaginated(query, "c", countQuery, parameters("id", id), pageable);
return neo4jClient.queryPaginated(query, "c", countQuery, parameters("id", id), pageable);
}


Expand Down
Loading
Loading