diff --git a/src/main/java/org/commcare/formplayer/database/models/FormplayerCaseIndexTable.java b/src/main/java/org/commcare/formplayer/database/models/FormplayerCaseIndexTable.java index 1a0461894..ef3bc7f5c 100644 --- a/src/main/java/org/commcare/formplayer/database/models/FormplayerCaseIndexTable.java +++ b/src/main/java/org/commcare/formplayer/database/models/FormplayerCaseIndexTable.java @@ -2,10 +2,13 @@ import static org.commcare.formplayer.sandbox.SqlSandboxUtils.execSql; +import com.google.common.collect.ImmutableMap; +import org.apache.commons.lang3.text.StrSubstitutor; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.commcare.cases.model.Case; import org.commcare.cases.model.CaseIndex; +import org.commcare.cases.query.queryset.DualTableMultiMatchModelQuerySet; import org.commcare.cases.query.queryset.DualTableSingleMatchModelQuerySet; import org.commcare.formplayer.sandbox.SqlHelper; import org.commcare.formplayer.sandbox.SqlStorage; @@ -26,6 +29,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Vector; +import java.util.function.Consumer; /** * @author ctsims @@ -54,7 +58,7 @@ public FormplayerCaseIndexTable(ConnectionHandler connectionHandler) { } public FormplayerCaseIndexTable(ConnectionHandler connectionHandler, String tableName, String caseTableName, - boolean createTable) { + boolean createTable) { this.connectionHandler = connectionHandler; this.tableName = tableName; this.caseTableName = caseTableName; @@ -284,51 +288,105 @@ public int loadIntoIndexTable(HashMap> indexCache, Strin * @return */ public DualTableSingleMatchModelQuerySet bulkReadIndexToCaseIdMatch(String indexName, - Collection cuedCases) { + Collection cuedCases) { DualTableSingleMatchModelQuerySet set = new DualTableSingleMatchModelQuerySet(); String caseIdIndex = TableBuilder.scrubName(Case.INDEX_CASE_ID); List> whereParamList = TableBuilder.sqlList(cuedCases, "?"); + + for (Pair querySet : whereParamList) { + + String query = String.format( + "SELECT %s,%s " + + "FROM %s " + + "INNER JOIN %s " + + "ON %s = %s " + + "WHERE %s = '%s' " + + "AND " + + "%s IN %s", + + COL_CASE_RECORD_ID, caseTableName + "." + DatabaseHelper.ID_COL, + getTableName(), + caseTableName, + COL_INDEX_TARGET, caseIdIndex, + COL_INDEX_NAME, indexName, + COL_CASE_RECORD_ID, querySet.first); + + executeQueryCollectResults(query, querySet.second, resultSet -> { + try { + int caseId = resultSet.getInt(resultSet.findColumn(COL_CASE_RECORD_ID)); + int targetCase = resultSet.getInt(resultSet.findColumn(DatabaseHelper.ID_COL)); + set.loadResult(caseId, targetCase); + } catch (SQLException e) { + throw new RuntimeException(e); + } + }); + } + return set; + } + + /** + * Performs a reverse index match on for the provided index name and target (parent) cases. + * + * @param indexName The name of the index e.g. 'parent' + * @param cuedCases Row IDs for the parent cases. + * @return ModelQuerySet with the results + */ + public DualTableMultiMatchModelQuerySet bulkReadCaseIdToIndexMatch(String indexName, + Collection cuedCases) { + DualTableMultiMatchModelQuerySet set = new DualTableMultiMatchModelQuerySet(); + String caseIdIndex = TableBuilder.scrubName(Case.INDEX_CASE_ID); + List> whereParamList = TableBuilder.sqlList(cuedCases, "?"); + + for (Pair querySet : whereParamList) { + + StrSubstitutor substitutor = new StrSubstitutor(ImmutableMap.of( + "caseId", caseTableName + "." + DatabaseHelper.ID_COL, + "childId", COL_CASE_RECORD_ID, + "caseTable", caseTableName, + "indexTable", getTableName(), + "targetCol", COL_INDEX_TARGET, + "caseIdCol", caseIdIndex, + "indexNameCol", COL_INDEX_NAME, + "indexName", indexName, + "parentCases", querySet.first + )); + String query = substitutor.replace("SELECT ${caseId}, ${childId} " + + "FROM ${caseTable} JOIN ${indexTable} ON ${targetCol} = ${caseIdCol} " + + "WHERE ${indexNameCol} = '${indexName}' AND ${caseId} IN ${parentCases};"); + + executeQueryCollectResults(query, querySet.second, resultSet -> { + try { + int caseId = resultSet.getInt(resultSet.findColumn(DatabaseHelper.ID_COL)); + int targetCase = resultSet.getInt(resultSet.findColumn(COL_CASE_RECORD_ID)); + set.loadResult(caseId, targetCase); + } catch (SQLException e) { + throw new RuntimeException(e); + } + }); + } + return set; + } + + private void executeQueryCollectResults(String query, String[] args, Consumer collector) { try { - for (Pair querySet : whereParamList) { - - String query = String.format( - "SELECT %s,%s " + - "FROM %s " + - "INNER JOIN %s " + - "ON %s = %s " + - "WHERE %s = '%s' " + - "AND " + - "%s IN %s", - - COL_CASE_RECORD_ID, caseTableName + "." + DatabaseHelper.ID_COL, - getTableName(), - caseTableName, - COL_INDEX_TARGET, caseIdIndex, - COL_INDEX_NAME, indexName, - COL_CASE_RECORD_ID, querySet.first); - - try (PreparedStatement preparedStatement = - connectionHandler.getConnection().prepareStatement(query)) { - int argIndex = 1; - for (String arg : querySet.second) { - preparedStatement.setString(argIndex, arg); - argIndex++; - } + try (PreparedStatement preparedStatement = + connectionHandler.getConnection().prepareStatement(query)) { + int argIndex = 1; + for (String arg : args) { + preparedStatement.setString(argIndex, arg); + argIndex++; + } - if (log.isTraceEnabled()) { - SqlHelper.explainSql(connectionHandler.getConnection(), query, querySet.second); - } + if (log.isTraceEnabled()) { + SqlHelper.explainSql(connectionHandler.getConnection(), query, args); + } - try (ResultSet resultSet = preparedStatement.executeQuery()) { - while (resultSet.next()) { - int caseId = resultSet.getInt(resultSet.findColumn(COL_CASE_RECORD_ID)); - int targetCase = resultSet.getInt(resultSet.findColumn(DatabaseHelper.ID_COL)); - set.loadResult(caseId, targetCase); - } + try (ResultSet resultSet = preparedStatement.executeQuery()) { + while (resultSet.next()) { + collector.accept(resultSet); } } } - return set; } catch (SQLException e) { throw new RuntimeException(e); } diff --git a/src/test/java/org/commcare/formplayer/tests/CaseDbModelQueryTests.java b/src/test/java/org/commcare/formplayer/tests/CaseDbModelQueryTests.java index d57b4797f..90546fc9a 100644 --- a/src/test/java/org/commcare/formplayer/tests/CaseDbModelQueryTests.java +++ b/src/test/java/org/commcare/formplayer/tests/CaseDbModelQueryTests.java @@ -1,43 +1,76 @@ package org.commcare.formplayer.tests; -import static org.commcare.formplayer.utils.DbTestUtils.evaluate; - +import com.google.common.collect.ImmutableMap; +import org.commcare.cases.query.QueryContext; +import org.commcare.cases.query.queryset.CurrentModelQuerySet; +import org.commcare.formplayer.application.UtilController; +import org.commcare.formplayer.configuration.CacheConfiguration; +import org.commcare.formplayer.junit.InitializeStaticsExtension; +import org.commcare.formplayer.junit.RestoreFactoryExtension; +import org.commcare.formplayer.junit.StorageFactoryExtension; +import org.commcare.formplayer.junit.request.SyncDbRequest; import org.commcare.formplayer.sandbox.UserSqlSandbox; +import org.commcare.formplayer.services.RestoreFactory; import org.commcare.formplayer.utils.TestContext; import org.commcare.formplayer.utils.TestStorageUtils; import org.javarosa.core.model.condition.EvaluationContext; +import org.javarosa.core.model.instance.TreeReference; +import org.javarosa.core.model.trace.AccumulatingReporter; +import org.javarosa.core.model.trace.EvaluationTraceReporter; +import org.javarosa.core.model.utils.InstrumentationUtils; +import org.javarosa.xpath.XPathLazyNodeset; +import org.javarosa.xpath.XPathParseTool; +import org.javarosa.xpath.expr.FunctionUtils; +import org.javarosa.xpath.parser.XPathSyntaxException; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.context.annotation.Import; import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.web.servlet.MockMvc; + +import java.util.function.Predicate; + +import static org.commcare.formplayer.utils.DbTestUtils.evaluate; + -/** - * @author wspride - */ @WebMvcTest -public class CaseDbModelQueryTests extends BaseTestClass { +@Import({UtilController.class}) +@ContextConfiguration(classes = {TestContext.class, CacheConfiguration.class}) +@ExtendWith(InitializeStaticsExtension.class) +public class CaseDbModelQueryTests { - @Override - @BeforeEach - public void setUp() throws Exception { - super.setUp(); - configureRestoreFactory("synctestdomain", "synctestusername"); - } + @Autowired + private MockMvc mockMvc; + + @Autowired + private RestoreFactory restoreFactory; - @Override - protected String getMockRestoreFileName() { - return "restores/dbtests/case_db_model_query.xml"; + @RegisterExtension + static RestoreFactoryExtension restoreFactoryExt = new RestoreFactoryExtension.builder() + .withUser("synctestusername").withDomain("synctestdomain") + .withRestorePath("restores/dbtests/case_db_model_query.xml") + .build(); + + @RegisterExtension + static StorageFactoryExtension storageExt = new StorageFactoryExtension.builder() + .withUser("back_nav").withDomain("back_nav").build(); + private EvaluationContext evaluationContext; + + @BeforeEach + public void setUp() { + new SyncDbRequest(mockMvc, restoreFactory).request(); + UserSqlSandbox sandbox = restoreFactory.getSqlSandbox(); + evaluationContext = TestStorageUtils.getEvaluationContextWithoutSession(sandbox); + evaluationContext.setDebugModeOn(); } - /** - * Tests for basic common case database queries - */ @Test - public void testDbModelQueryLookups() throws Exception { - syncDb(); - UserSqlSandbox sandbox = getRestoreSandbox(); - EvaluationContext ec = TestStorageUtils.getEvaluationContextWithoutSession(sandbox); - ec.setDebugModeOn(); + public void testModelQueryLookupDerivations() throws Exception { evaluate( "join(',',instance('casedb')/casedb/case[@case_type='unit_test_child_child" + "'][@status='open'][true() and " @@ -45,7 +78,11 @@ public void testDbModelQueryLookups() throws Exception { "instance('casedb')/casedb/case[@case_id = instance('casedb')" + "/casedb/case[@case_id=current()/index/parent]/index/parent]/test = " + "'true']/@case_id)", - "child_ptwo_one_one,child_one_one", ec); + "child_ptwo_one_one,child_one_one", evaluationContext); + } + + @Test + public void testModelSelfReference() throws XPathSyntaxException { evaluate( "join(',',instance('casedb')/casedb/case[@case_type='unit_test_child'][@status" + "='open'][true() and " @@ -53,7 +90,88 @@ public void testDbModelQueryLookups() throws Exception { "count(instance('casedb')/casedb/case[index/parent = instance('casedb')" + "/casedb/case[@case_id=current()/@case_id]/index/parent][false = " + "'true']) > 0]/@case_id)", - "", ec); + "", evaluationContext); + + } + + @Test + public void testModelReverseIndexLookup() throws XPathSyntaxException { + XPathLazyNodeset nodeset = (XPathLazyNodeset) XPathParseTool.parseXPath( + "instance('casedb')/casedb/case[@case_type='unit_test_parent']").eval(evaluationContext); + + ImmutableMap expectedOutputs = ImmutableMap.of( + "test_case_parent", "child_one,child_two,child_three", + "parent_two", "child_ptwo_one" + ); + Assertions.assertEquals(nodeset.getReferences().size(), expectedOutputs.size()); + + EvaluationTraceReporter reporter = new AccumulatingReporter(); + evaluationContext.setDebugModeOn(reporter); + for (TreeReference current : nodeset.getReferences()) { + EvaluationContext subContext = new EvaluationContext(evaluationContext, current); + QueryContext newContext = subContext.getCurrentQueryContext() + .checkForDerivativeContextAndReturn(nodeset.getReferences().size()); + newContext.setHackyOriginalContextBody(new CurrentModelQuerySet(nodeset.getReferences())); + subContext.setQueryContext(newContext); + + String parentCaseId = FunctionUtils.toString( + XPathParseTool.parseXPath("./@case_id").eval(subContext) + ); + + evaluate( + "join(',',instance('casedb')/casedb/case[index/parent = current()/@case_id]/@case_id)", + expectedOutputs.get(parentCaseId), + subContext + ); + + } + Predicate predicate = line -> line.contains("Load Query Set Transform[current]=>[current|reverse index|parent]: Loaded: 4"); + int matchedReverseIndexLoad = InstrumentationUtils.countMatchedTraces(reporter, predicate); + Assertions.assertEquals(1, matchedReverseIndexLoad); + + int matchedLookups = InstrumentationUtils.countMatchedTraces(reporter, line -> line.contains("QuerySetLookup|current|reverse index|parent: Results:")); + Assertions.assertEquals(2, matchedLookups); + } + + @Test + public void testModelIndexLookup() throws XPathSyntaxException { + XPathLazyNodeset nodeset = (XPathLazyNodeset) XPathParseTool.parseXPath( + "instance('casedb')/casedb/case[@case_type='unit_test_child']").eval(evaluationContext); + EvaluationTraceReporter reporter = new AccumulatingReporter(); + evaluationContext.setDebugModeOn(reporter); + + ImmutableMap expectedOutputs = ImmutableMap.of( + "child_one", "test_case_parent", + "child_two", "test_case_parent", + "child_three", "test_case_parent", + "child_ptwo_one", "parent_two" + ); + + Assertions.assertEquals(nodeset.getReferences().size(), expectedOutputs.size()); + + for (TreeReference current : nodeset.getReferences()) { + EvaluationContext subContext = new EvaluationContext(evaluationContext, current); + QueryContext newContext = subContext.getCurrentQueryContext() + .checkForDerivativeContextAndReturn(nodeset.getReferences().size()); + newContext.setHackyOriginalContextBody(new CurrentModelQuerySet(nodeset.getReferences())); + subContext.setQueryContext(newContext); + + String childCaseId = FunctionUtils.toString( + XPathParseTool.parseXPath("./@case_id").eval(subContext) + ); + + evaluate( + "join(',',instance('casedb')/casedb/case[@case_id = current()/index/parent]/@case_id)", + expectedOutputs.get(childCaseId), + subContext + ); + + } + Predicate predicate = line -> line.contains("Load Query Set Transform[current]=>[current|index|parent]: Loaded: 2"); + int matchedIndexLoad = InstrumentationUtils.countMatchedTraces(reporter, predicate); + Assertions.assertEquals(1, matchedIndexLoad); + int matchedLookups = InstrumentationUtils.countMatchedTraces(reporter, line -> line.contains("QuerySetLookup|current|index|parent: Results: 1")); + Assertions.assertEquals(4, matchedLookups); } } diff --git a/src/test/java/org/commcare/formplayer/tests/CaseDbOptimizationsTest.java b/src/test/java/org/commcare/formplayer/tests/CaseDbOptimizationsTest.java index e2f6d1e39..ccf7d65b4 100644 --- a/src/test/java/org/commcare/formplayer/tests/CaseDbOptimizationsTest.java +++ b/src/test/java/org/commcare/formplayer/tests/CaseDbOptimizationsTest.java @@ -1,73 +1,101 @@ package org.commcare.formplayer.tests; -import static org.commcare.formplayer.utils.DbTestUtils.evaluate; - +import org.commcare.formplayer.application.UtilController; +import org.commcare.formplayer.configuration.CacheConfiguration; +import org.commcare.formplayer.junit.InitializeStaticsExtension; +import org.commcare.formplayer.junit.RestoreFactoryExtension; +import org.commcare.formplayer.junit.StorageFactoryExtension; +import org.commcare.formplayer.junit.request.SyncDbRequest; import org.commcare.formplayer.sandbox.UserSqlSandbox; +import org.commcare.formplayer.services.RestoreFactory; import org.commcare.formplayer.utils.TestContext; import org.commcare.formplayer.utils.TestStorageUtils; import org.javarosa.core.model.condition.EvaluationContext; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.context.annotation.Import; import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.web.servlet.MockMvc; + +import static org.commcare.formplayer.utils.DbTestUtils.evaluate; -/** - * @author wspride - */ @WebMvcTest -public class CaseDbOptimizationsTest extends BaseTestClass { +@Import({UtilController.class}) +@ContextConfiguration(classes = {TestContext.class, CacheConfiguration.class}) +@ExtendWith(InitializeStaticsExtension.class) +public class CaseDbOptimizationsTest { - @Override - @BeforeEach - public void setUp() throws Exception { - super.setUp(); - configureRestoreFactory("synctestdomain", "synctestusername"); - } + @Autowired + private MockMvc mockMvc; - @Override - protected String getMockRestoreFileName() { - return "restores/dbtests/case_test_db_optimizations.xml"; - } + @Autowired + private RestoreFactory restoreFactory; + + @RegisterExtension + static RestoreFactoryExtension restoreFactoryExt = new RestoreFactoryExtension.builder() + .withUser("synctestusername").withDomain("synctestdomain") + .withRestorePath("restores/dbtests/case_test_db_optimizations.xml") + .build(); + @RegisterExtension + static StorageFactoryExtension storageExt = new StorageFactoryExtension.builder() + .withUser("back_nav").withDomain("back_nav").build(); + private UserSqlSandbox sandbox; + private EvaluationContext evaluationContext; + + @BeforeEach + public void setUp() { + new SyncDbRequest(mockMvc, restoreFactory).request(); + sandbox = restoreFactory.getSqlSandbox(); + evaluationContext = TestStorageUtils.getEvaluationContextWithoutSession(sandbox); + evaluationContext.setDebugModeOn(); + } /** * Tests for basic common case database queries */ @Test public void testDbOptimizations() throws Exception { - syncDb(); - UserSqlSandbox sandbox = getRestoreSandbox(); EvaluationContext ec = TestStorageUtils.getEvaluationContextWithoutSession(sandbox); ec.setDebugModeOn(); evaluate( "sort(join(' ',instance('casedb')/casedb/case[index/parent = " + "'test_case_parent']/@case_id))", - "child_one child_three child_two", ec); + "child_one child_three child_two", evaluationContext); evaluate( "join(' ',instance('casedb')/casedb/case[index/parent = " + "'test_case_parent'][@case_id = 'child_two']/@case_id)", - "child_two", ec); + "child_two", evaluationContext); evaluate( "sort(join(' ',instance('casedb')/casedb/case[index/parent = " + "'test_case_parent'][@case_id != 'child_two']/@case_id))", - "child_one child_three", ec); + "child_one child_three", evaluationContext); + } + @Test + public void testSelectedOptimization() throws Exception { + EvaluationContext ec = TestStorageUtils.getEvaluationContextWithoutSession(sandbox); + ec.setDebugModeOn(); evaluate( "sort(join(' ',instance('casedb')/casedb/case[selected('test_case_parent', " + "index/parent)]/@case_id))", - "child_one child_three child_two", ec); + "child_one child_three child_two", evaluationContext); evaluate( "sort(join(' ',instance('casedb')/casedb/case[selected('test_case_parent " + "test_case_parent_2', index/parent)]/@case_id))", - "child_one child_three child_two", ec); + "child_one child_three child_two", evaluationContext); evaluate( "sort(join(' ',instance('casedb')/casedb/case[selected('test_case_parent_2 " + "test_case_parent', index/parent)]/@case_id))", - "child_one child_three child_two", ec); + "child_one child_three child_two", evaluationContext); evaluate( "join(' ',instance('casedb')/casedb/case[selected('test_case_parent_2 " + "test_case_parent_3', index/parent)]/@case_id)", - "", ec); + "", evaluationContext); evaluate("join(' ',instance('casedb')/casedb/case[selected('', index/parent)]/@case_id)", - "", ec); + "", evaluationContext); } } diff --git a/src/test/java/org/commcare/formplayer/utils/TestContext.java b/src/test/java/org/commcare/formplayer/utils/TestContext.java index 8a4beb4e5..9100ce383 100644 --- a/src/test/java/org/commcare/formplayer/utils/TestContext.java +++ b/src/test/java/org/commcare/formplayer/utils/TestContext.java @@ -7,22 +7,7 @@ import org.commcare.formplayer.mocks.MockLockRegistry; import org.commcare.formplayer.mocks.TestInstallService; import org.commcare.formplayer.objects.FormVolatilityRecord; -import org.commcare.formplayer.services.CaseSearchHelper; -import org.commcare.formplayer.services.CategoryTimingHelper; -import org.commcare.formplayer.services.FormDefinitionService; -import org.commcare.formplayer.services.FormSessionService; -import org.commcare.formplayer.services.FormplayerFormSendCalloutHandler; -import org.commcare.formplayer.services.FormplayerStorageFactory; -import org.commcare.formplayer.services.HqUserDetailsService; -import org.commcare.formplayer.services.InstallService; -import org.commcare.formplayer.services.MediaMetaDataService; -import org.commcare.formplayer.services.MenuSessionFactory; -import org.commcare.formplayer.services.MenuSessionRunnerService; -import org.commcare.formplayer.services.MenuSessionService; -import org.commcare.formplayer.services.NewFormResponseFactory; -import org.commcare.formplayer.services.RestoreFactory; -import org.commcare.formplayer.services.SubmitService; -import org.commcare.formplayer.services.VirtualDataInstanceService; +import org.commcare.formplayer.services.*; import org.commcare.formplayer.util.Constants; import org.commcare.formplayer.util.FormplayerDatadog; import org.commcare.formplayer.util.NotificationLogger; @@ -103,6 +88,9 @@ public InternalResourceViewResolver viewResolver() { @MockBean public NotificationLogger notificationLogger; + @MockBean + public FormplayerLockRegistry userLockRegistry; + @Bean public ValueOperations redisTemplateLong() { return Mockito.mock(ValueOperations.class); @@ -153,11 +141,6 @@ public FormplayerDatadog datadog() { return Mockito.spy(new FormplayerDatadog(datadogStatsDClient(), new ArrayList())); } - @Bean - public LockRegistry userLockRegistry() { - return Mockito.spy(MockLockRegistry.class); - } - @Bean public NewFormResponseFactory newFormResponseFactory() { return Mockito.spy(NewFormResponseFactory.class);