diff --git a/dao-api/src/main/java/com/linkedin/metadata/dao/BaseLocalDAO.java b/dao-api/src/main/java/com/linkedin/metadata/dao/BaseLocalDAO.java index d9ec98021..a3c3b00de 100644 --- a/dao-api/src/main/java/com/linkedin/metadata/dao/BaseLocalDAO.java +++ b/dao-api/src/main/java/com/linkedin/metadata/dao/BaseLocalDAO.java @@ -380,8 +380,8 @@ protected abstract long saveLatest(@Nonnull URN * @param newValue {@link RecordTemplate} of the new value of aspect * @param version version of the aspect */ - public abstract void updateLocalIndex(@Nonnull URN urn, - @Nullable ASPECT newValue, long version); + public abstract void updateLocalIndex(@Nonnull URN urn, @Nullable ASPECT newValue, + long version); /** * Returns list of urns from local secondary index that satisfy the given filter conditions. @@ -407,34 +407,37 @@ public List listUrns(@Nonnull IndexFilter indexFilter, @Nullable URN lastUr return listUrns(indexFilter, null, lastUrn, pageSize); } + /** + * Similar to {@link #listUrns(IndexFilter, IndexSortCriterion, Urn, int)} but returns a list result with pagination + * information. + * + * @param start the starting offset of the page + * @return a {@link ListResult} containing a list of urns and other pagination information + */ + @Nonnull + public abstract ListResult listUrns(@Nonnull IndexFilter indexFilter, + @Nullable IndexSortCriterion indexSortCriterion, int start, int pageSize); + /** * Similar to {@link #listUrns(IndexFilter, Urn, int)}. This is to get all urns with type URN. */ @Nonnull public List listUrns(@Nonnull Class urnClazz, @Nullable URN lastUrn, int pageSize) { - final IndexFilter indexFilter = new IndexFilter() - .setCriteria(new IndexCriterionArray(new IndexCriterion().setAspect(urnClazz.getCanonicalName()))); + final IndexFilter indexFilter = new IndexFilter().setCriteria( + new IndexCriterionArray(new IndexCriterion().setAspect(urnClazz.getCanonicalName()))); return listUrns(indexFilter, lastUrn, pageSize); } /** - * Retrieves list of {@link UrnAspectEntry} containing latest version of aspects along with the urn for the list of urns - * returned from local secondary index that satisfy given filter conditions. The returned list is ordered by the - * sort criterion but ordered lexicographically by the string representation of the URN by default. + * Retrieves list of urn aspect entries corresponding to the aspect classes and urns. * * @param aspectClasses aspect classes whose latest versions need to be retrieved - * @param indexFilter {@link IndexFilter} containing filter conditions to be applied - * @param indexSortCriterion {@link IndexSortCriterion} sort conditions to be applied - * @param lastUrn last urn of the previous fetched page. For the first page, this should be set as NULL - * @param pageSize maximum number of distinct urns whose aspects need to be retrieved - * @return ordered list of latest versions of aspects along with urns returned from local secondary index - * satisfying given filter conditions + * @param urns corresponding urns to be retrieved + * @return list of latest versions of aspects along with urns */ @Nonnull - public List> getAspects(@Nonnull Set> aspectClasses, - @Nonnull IndexFilter indexFilter, @Nullable IndexSortCriterion indexSortCriterion, @Nullable URN lastUrn, int pageSize) { - - final List urns = listUrns(indexFilter, indexSortCriterion, lastUrn, pageSize); + private List> getUrnAspectEntries(@Nonnull Set> aspectClasses, + @Nonnull List urns) { final Map, Optional>> urnAspectMap = get(aspectClasses, new HashSet<>(urns)); @@ -458,16 +461,66 @@ public List> getAspects(@Nonnull Set> getAspects(@Nonnull Set> aspectClasses, + @Nonnull IndexFilter indexFilter, @Nullable IndexSortCriterion indexSortCriterion, @Nullable URN lastUrn, + int pageSize) { + + final List urns = listUrns(indexFilter, indexSortCriterion, lastUrn, pageSize); + + return getUrnAspectEntries(aspectClasses, urns); + } + /** * Similar to {@link #getAspects(Set, IndexFilter, IndexSortCriterion, Urn, int)} * but sorts lexicographically by the URN. */ @Nonnull public List> getAspects(@Nonnull Set> aspectClasses, - @Nonnull IndexFilter indexFilter, @Nullable URN lastUrn, int pageSize) { + @Nonnull IndexFilter indexFilter, @Nullable URN lastUrn, int pageSize) { return getAspects(aspectClasses, indexFilter, null, lastUrn, pageSize); } + /** + * Similar to {@link #getAspects(Set, IndexFilter, IndexSortCriterion, Urn, int)} + * but returns a list of aspects with pagination information. + * + * @param start starting offset of the page + * @return a {@link ListResult} containing an ordered list of latest versions of aspects along with urns returned from + * local secondary index satisfying given filter conditions and pagination information + */ + @Nonnull + public ListResult> getAspects(@Nonnull Set> aspectClasses, + @Nonnull IndexFilter indexFilter, @Nullable IndexSortCriterion indexSortCriterion, int start, int pageSize) { + + final ListResult listResult = listUrns(indexFilter, indexSortCriterion, start, pageSize); + final List urns = listResult.getValues(); + + final List> urnAspectEntries = getUrnAspectEntries(aspectClasses, urns); + + return ListResult.>builder().values(urnAspectEntries) + .metadata(listResult.getMetadata()) + .nextStart(listResult.getNextStart()) + .havingMore(listResult.isHavingMore()) + .totalCount(listResult.getTotalCount()) + .totalPageCount(listResult.getTotalPageCount()) + .pageSize(listResult.getPageSize()) + .build(); + } + /** * Runs the given lambda expression in a transaction with a limited number of retries. * @@ -544,7 +597,8 @@ protected abstract void applyTimeBasedRetention( * @deprecated Use {@link #backfill(Set, Set)} instead */ @Nonnull - public Optional backfill(@Nonnull Class aspectClass, @Nonnull URN urn) { + public Optional backfill(@Nonnull Class aspectClass, + @Nonnull URN urn) { return backfill(BackfillMode.BACKFILL_ALL, aspectClass, urn); } @@ -584,7 +638,8 @@ public Map, Optional, Optional>> backfill( @Nonnull BackfillMode mode, @Nonnull Set> aspectClasses, @Nonnull Set urns) { checkValidAspects(aspectClasses); - final Map, Optional>> urnToAspects = get(aspectClasses, urns); + final Map, Optional>> urnToAspects = + get(aspectClasses, urns); urnToAspects.forEach((urn, aspects) -> { aspects.forEach((aspectClass, aspect) -> aspect.ifPresent(value -> backfill(mode, value, urn))); }); @@ -619,7 +674,8 @@ public Map, Optional must be a supported aspect type in {@code ASPECT_UNION}. */ - private void backfill(@Nonnull BackfillMode mode, @Nonnull ASPECT aspect, @Nonnull URN urn) { + private void backfill(@Nonnull BackfillMode mode, @Nonnull ASPECT aspect, + @Nonnull URN urn) { if (_enableLocalSecondaryIndex && (mode == BackfillMode.SCSI_ONLY || mode == BackfillMode.BACKFILL_ALL)) { updateLocalIndex(urn, aspect, FIRST_VERSION); } diff --git a/dao-api/src/test/java/com/linkedin/metadata/dao/BaseLocalDAOTest.java b/dao-api/src/test/java/com/linkedin/metadata/dao/BaseLocalDAOTest.java index 225fb4d79..856529b9d 100644 --- a/dao-api/src/test/java/com/linkedin/metadata/dao/BaseLocalDAOTest.java +++ b/dao-api/src/test/java/com/linkedin/metadata/dao/BaseLocalDAOTest.java @@ -51,8 +51,8 @@ protected long saveLatest(FooUrn urn, Class void updateLocalIndex(@Nonnull FooUrn urn, - @Nullable ASPECT newValue, long version) { + public void updateLocalIndex(@Nonnull FooUrn urn, @Nullable ASPECT newValue, + long version) { } @@ -106,6 +106,12 @@ public List listUrns(@Nonnull IndexFilter indexFilter, @Nullable IndexSo return null; } + @Override + public ListResult listUrns(@Nonnull IndexFilter indexFilter, + @Nullable IndexSortCriterion indexSortCriterion, int start, int pageSize) { + return ListResult.builder().build(); + } + @Override public ListResult list(Class aspectClass, FooUrn urn, int start, int pageSize) { diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java index 9bd3ba15b..c3c180b33 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java @@ -248,7 +248,7 @@ public static ServerConfig createTestingH2ServerConfig() { DataSourceConfig dataSourceConfig = new DataSourceConfig(); dataSourceConfig.setUsername("tester"); dataSourceConfig.setPassword(""); - dataSourceConfig.setUrl("jdbc:h2:mem:;IGNORECASE=TRUE;"); + dataSourceConfig.setUrl("jdbc:h2:mem:testdb;IGNORECASE=TRUE;"); dataSourceConfig.setDriver("org.h2.Driver"); ServerConfig serverConfig = new ServerConfig(); @@ -298,7 +298,7 @@ protected long saveLatest(@Nonnull URN urn, @Non // update latest version if (_useOptimisticLocking) { saveWithOptimisticLocking(urn, newValue, newAuditStamp, LATEST_VERSION, false, - new Timestamp(oldAuditStamp.getTime())); + new Timestamp(oldAuditStamp.getTime())); } else { save(urn, newValue, newAuditStamp, LATEST_VERSION, false); } @@ -339,7 +339,7 @@ protected AspectEntry getLatest(@Nonnull @Nonnull private EbeanMetadataAspect buildMetadataAspectBean(@Nonnull URN urn, @Nonnull RecordTemplate value, - @Nonnull AuditStamp auditStamp, long version) { + @Nonnull AuditStamp auditStamp, long version) { final String aspectName = ModelUtils.getAspectName(value.getClass()); @@ -359,9 +359,7 @@ private EbeanMetadataAspect buildMetadataAspectBean(@Nonnull URN urn, @Nonnull R // visible for testing protected void saveWithOptimisticLocking(@Nonnull URN urn, @Nonnull RecordTemplate value, - @Nonnull AuditStamp newAuditStamp, - long version, boolean insert, - @Nonnull Object oldTimestamp) { + @Nonnull AuditStamp newAuditStamp, long version, boolean insert, @Nonnull Object oldTimestamp) { final EbeanMetadataAspect aspect = buildMetadataAspectBean(urn, value, newAuditStamp, version); @@ -379,8 +377,8 @@ protected void saveWithOptimisticLocking(@Nonnull URN urn, @Nonnull RecordTempla // Ideally, another column for the sake of optimistic locking would be preferred but that means a change to // metadata_aspect schema and we don't take this route here to keep this change backward compatible. final String updateQuery = String.format("UPDATE metadata_aspect " - + "SET urn = :urn, aspect = :aspect, version = :version, metadata = :metadata, createdOn = :createdOn, createdBy = :createdBy " - + "WHERE urn = :urn and aspect = :aspect and version = :version and createdOn = :oldTimestamp"); + + "SET urn = :urn, aspect = :aspect, version = :version, metadata = :metadata, createdOn = :createdOn, createdBy = :createdBy " + + "WHERE urn = :urn and aspect = :aspect and version = :version and createdOn = :oldTimestamp"); final SqlUpdate update = _server.createSqlUpdate(updateQuery); update.setParameter("urn", aspect.getKey().getUrn()); @@ -394,7 +392,8 @@ protected void saveWithOptimisticLocking(@Nonnull URN urn, @Nonnull RecordTempla // If there is no single updated row, emit OptimisticLockException int numOfUpdatedRows = _server.execute(update); if (numOfUpdatedRows != 1) { - throw new OptimisticLockException(numOfUpdatedRows + " rows updated during save query: " + update.getGeneratedSql()); + throw new OptimisticLockException( + numOfUpdatedRows + " rows updated during save query: " + update.getGeneratedSql()); } } @@ -450,8 +449,7 @@ private void updateUrnInLocalIndex(@Nonnull URN urn) { } final Map pathValueMap = _urnPathExtractor.extractPaths(urn); - pathValueMap.forEach( - (path, value) -> saveSingleRecordToLocalIndex(urn, _urnClass.getCanonicalName(), path, value)); + pathValueMap.forEach((path, value) -> saveSingleRecordToLocalIndex(urn, _urnClass.getCanonicalName(), path, value)); } private void updateAspectInLocalIndex(@Nonnull URN urn, @Nonnull ASPECT newValue) { @@ -909,7 +907,8 @@ static GMAIndexPair getGMAIndexPair(@Nonnull IndexCriterion criterion) { } else if (indexValue.isString()) { object = getValueFromIndexCriterion(criterion); return new GMAIndexPair(EbeanMetadataIndex.STRING_COLUMN, object); - } else if (indexValue.isArray() && indexValue.getArray().size() > 0 && indexValue.getArray().get(0).getClass() == String.class) { + } else if (indexValue.isArray() && indexValue.getArray().size() > 0 + && indexValue.getArray().get(0).getClass() == String.class) { object = indexValue.getArray(); return new GMAIndexPair(EbeanMetadataIndex.STRING_COLUMN, object); } else { @@ -928,6 +927,7 @@ static String getValueFromIndexCriterion(@Nonnull IndexCriterion criterion) { /** * Sets the values of parameters in metadata index query based on its position, values obtained from * {@link IndexCriterionArray} and last urn. Also sets the LIMIT of SQL query using the page size input. + * For offset pagination, the limit will be set when the query gets executed. * * @param indexCriterionArray {@link IndexCriterionArray} whose values will be used to set parameters in metadata * index query based on its position @@ -935,11 +935,15 @@ static String getValueFromIndexCriterion(@Nonnull IndexCriterion criterion) { * @param indexQuery {@link Query} whose ordered parameters need to be set, based on it's position * @param lastUrn string representation of the urn whose value is used to set the last urn parameter in index query * @param pageSize maximum number of distinct urns to return which is essentially the LIMIT clause of SQL query + * @param offsetPagination used to determine whether to used cursor or offset pagination */ - private static void setParameters(@Nonnull IndexCriterionArray indexCriterionArray, @Nullable IndexSortCriterion indexSortCriterion, - @Nonnull Query indexQuery, @Nonnull String lastUrn, int pageSize) { - indexQuery.setParameter(1, lastUrn); - int pos = 2; + private static void setParameters(@Nonnull IndexCriterionArray indexCriterionArray, + @Nullable IndexSortCriterion indexSortCriterion, @Nonnull Query indexQuery, + @Nonnull String lastUrn, int pageSize, boolean offsetPagination) { + int pos = 1; + if (!offsetPagination) { + indexQuery.setParameter(pos++, lastUrn); + } for (IndexCriterion criterion : indexCriterionArray) { indexQuery.setParameter(pos++, criterion.getAspect()); if (criterion.getPathParams() != null) { @@ -951,7 +955,9 @@ private static void setParameters(@Nonnull IndexCriterionArray indexCriterionArr indexQuery.setParameter(pos++, indexSortCriterion.getAspect()); indexQuery.setParameter(pos++, indexSortCriterion.getPath()); } - indexQuery.setParameter(pos, pageSize); + if (!offsetPagination) { + indexQuery.setParameter(pos, pageSize); + } } @Nonnull @@ -1009,7 +1015,8 @@ static String getSortingColumn(@Nonnull IndexSor } else if (type == DataSchema.Type.STRING || type == DataSchema.Type.BOOLEAN || type == DataSchema.Type.ENUM) { return EbeanMetadataIndex.STRING_COLUMN; } else { - throw new UnsupportedOperationException("The type stored in the path field of the aspect can not be sorted on" + indexSortCriterion); + throw new UnsupportedOperationException( + "The type stored in the path field of the aspect can not be sorted on" + indexSortCriterion); } } @@ -1027,14 +1034,17 @@ private static String getPlaceholderStringForValue(@Nonnull IndexValue indexValu /** * Constructs SQL query that contains positioned parameters (with `?`), based on whether {@link IndexCriterion} of * a given condition has field `pathParams`. + * For offset pagination, the limit clause is empty because the limit will be set when the query + * gets executed. * * @param indexCriterionArray {@link IndexCriterionArray} used to construct the SQL query * @param indexSortCriterion {@link IndexSortCriterion} used to construct the SQL query + * @param offsetPagination used to determine whether to used cursor or offset pagination * @return String representation of SQL query */ @Nonnull private static String constructSQLQuery(@Nonnull IndexCriterionArray indexCriterionArray, - @Nullable IndexSortCriterion indexSortCriterion) { + @Nullable IndexSortCriterion indexSortCriterion, boolean offsetPagination) { String sortColumn = indexSortCriterion != null ? getSortingColumn(indexSortCriterion) : ""; String selectClause = "SELECT DISTINCT(t0.urn)"; if (!sortColumn.isEmpty()) { @@ -1045,11 +1055,16 @@ private static String constructSQLQuery(@Nonnull IndexCriterionArray indexCriter selectClause += IntStream.range(1, indexCriterionArray.size()) .mapToObj(i -> " INNER JOIN metadata_index " + "t" + i + " ON t0.urn = " + "t" + i + ".urn") .collect(Collectors.joining("")); - final StringBuilder whereClause = new StringBuilder("WHERE t0.urn > ?"); + final StringBuilder whereClause = new StringBuilder("WHERE "); + if (!offsetPagination) { + whereClause.append("t0.urn > ?"); + } IntStream.range(0, indexCriterionArray.size()).forEach(i -> { final IndexCriterion criterion = indexCriterionArray.get(i); - - whereClause.append(" AND t").append(i).append(".aspect = ?"); + if (!offsetPagination || i > 0) { + whereClause.append(" AND"); + } + whereClause.append(" t").append(i).append(".aspect = ?"); if (criterion.getPathParams() != null) { validateConditionAndValue(criterion); whereClause.append(" AND t") @@ -1073,10 +1088,20 @@ private static String constructSQLQuery(@Nonnull IndexCriterionArray indexCriter } else { orderByClause = "ORDER BY urn ASC"; } - final String limitClause = "LIMIT ?"; + final String limitClause = offsetPagination ? "" : "LIMIT ?"; return String.join(" ", selectClause, whereClause, orderByClause, limitClause); } + void checkValidIndexCriterionArray(@Nonnull IndexCriterionArray indexCriterionArray) { + if (indexCriterionArray.isEmpty()) { + throw new UnsupportedOperationException("Empty Index Filter is not supported by EbeanLocalDAO"); + } + if (indexCriterionArray.size() > 10) { + throw new UnsupportedOperationException( + "Currently more than 10 filter conditions is not supported by EbeanLocalDAO"); + } + } + void addEntityTypeFilter(@Nonnull IndexFilter indexFilter) { if (indexFilter.getCriteria().stream().noneMatch(x -> x.getAspect().equals(_urnClass.getCanonicalName()))) { indexFilter.getCriteria().add(new IndexCriterion().setAspect(_urnClass.getCanonicalName())); @@ -1104,24 +1129,55 @@ public List listUrns(@Nonnull IndexFilter indexFilter, @Nullable IndexSortC if (!isLocalSecondaryIndexEnabled()) { throw new UnsupportedOperationException("Local secondary index isn't supported"); } + final IndexCriterionArray indexCriterionArray = indexFilter.getCriteria(); - if (indexCriterionArray.isEmpty()) { - throw new UnsupportedOperationException("Empty Index Filter is not supported by EbeanLocalDAO"); - } - if (indexCriterionArray.size() > 10) { - throw new UnsupportedOperationException( - "Currently more than 10 filter conditions is not supported by EbeanLocalDAO"); - } + checkValidIndexCriterionArray(indexCriterionArray); addEntityTypeFilter(indexFilter); final Query query = - _server.findNative(EbeanMetadataIndex.class, constructSQLQuery(indexCriterionArray, indexSortCriterion)) + _server.findNative(EbeanMetadataIndex.class, constructSQLQuery(indexCriterionArray, indexSortCriterion, false)) .setTimeout(INDEX_QUERY_TIMEOUT_IN_SEC); - setParameters(indexCriterionArray, indexSortCriterion, query, lastUrn == null ? "" : lastUrn.toString(), pageSize); + setParameters(indexCriterionArray, indexSortCriterion, query, lastUrn == null ? "" : lastUrn.toString(), pageSize, + false); final List pagedList = query.findList(); return pagedList.stream().map(entry -> getUrn(entry.getUrn())).collect(Collectors.toList()); } + + /** + * Similar to {@link #listUrns(IndexFilter, IndexSortCriterion, Urn, int)} but returns a list result with pagination + * information. + * + * @param start the starting offset of the page + * @return a {@link ListResult} containing a list of urns and other pagination information + */ + @Override + @Nonnull + public ListResult listUrns(@Nonnull IndexFilter indexFilter, @Nullable IndexSortCriterion indexSortCriterion, + int start, int pageSize) { + if (!isLocalSecondaryIndexEnabled()) { + throw new UnsupportedOperationException("Local secondary index isn't supported"); + } + + final IndexCriterionArray indexCriterionArray = indexFilter.getCriteria(); + checkValidIndexCriterionArray(indexCriterionArray); + + addEntityTypeFilter(indexFilter); + + final Query query = + _server.findNative(EbeanMetadataIndex.class, constructSQLQuery(indexCriterionArray, indexSortCriterion, true)) + .setTimeout(INDEX_QUERY_TIMEOUT_IN_SEC); + setParameters(indexCriterionArray, indexSortCriterion, query, "", pageSize, true); + + final PagedList pagedList = query.setFirstRow(start).setMaxRows(pageSize).findPagedList(); + + pagedList.loadCount(); + + final List urns = + pagedList.getList().stream().map(entry -> getUrn(entry.getUrn())).collect(Collectors.toList()); + + return toListResult(urns, null, pagedList, start); + } } \ No newline at end of file diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/EbeanLocalDAOTest.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/EbeanLocalDAOTest.java index 0382a1c4c..2c6011c1f 100644 --- a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/EbeanLocalDAOTest.java +++ b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/EbeanLocalDAOTest.java @@ -92,9 +92,7 @@ public EbeanLocalDAOTest(boolean useUnionForBatch, boolean useOptimisticLocking) @DataProvider public static Object[][] inputList() { - return new Object[][]{ - {false, false}, {true, true} - }; + return new Object[][]{{false, false}, {true, true}}; } @BeforeMethod @@ -452,7 +450,6 @@ public void testLocalSecondaryIndexBackfillDisabled() { assertEquals(getAllRecordsFromLocalIndex(urn).size(), 0); } - @Test public void testLocalSecondaryIndexBackfillEnabled() { // given @@ -856,63 +853,71 @@ public void testStartsWith() { public void testGetSortingColumn() { // 1. string corresponds to string column IndexSortCriterion indexSortCriterion1 = new IndexSortCriterion().setAspect(AspectFoo.class.getCanonicalName()) - .setPath("/value").setOrder(SortOrder.DESCENDING); + .setPath("/value") + .setOrder(SortOrder.DESCENDING); String sortingColumn1 = EbeanLocalDAO.getSortingColumn(indexSortCriterion1); assertEquals(sortingColumn1, EbeanMetadataIndex.STRING_COLUMN); // 2. boolean corresponds to string column IndexSortCriterion indexSortCriterion2 = new IndexSortCriterion().setAspect(AspectBaz.class.getCanonicalName()) - .setPath("/boolField").setOrder(SortOrder.DESCENDING); + .setPath("/boolField") + .setOrder(SortOrder.DESCENDING); String sortingColumn2 = EbeanLocalDAO.getSortingColumn(indexSortCriterion2); assertEquals(sortingColumn2, EbeanMetadataIndex.STRING_COLUMN); // 3. int corresponds to long column IndexSortCriterion indexSortCriterion3 = new IndexSortCriterion().setAspect(AspectBaz.class.getCanonicalName()) - .setPath("/intField").setOrder(SortOrder.DESCENDING); + .setPath("/intField") + .setOrder(SortOrder.DESCENDING); String sortingColumn3 = EbeanLocalDAO.getSortingColumn(indexSortCriterion3); assertEquals(sortingColumn3, EbeanMetadataIndex.LONG_COLUMN); // 4. long corresponds to long column IndexSortCriterion indexSortCriterion4 = new IndexSortCriterion().setAspect(AspectBaz.class.getCanonicalName()) - .setPath("/longField").setOrder(SortOrder.DESCENDING); + .setPath("/longField") + .setOrder(SortOrder.DESCENDING); String sortingColumn4 = EbeanLocalDAO.getSortingColumn(indexSortCriterion4); assertEquals(sortingColumn4, EbeanMetadataIndex.LONG_COLUMN); // 5. double corresponds to double column IndexSortCriterion indexSortCriterion5 = new IndexSortCriterion().setAspect(AspectBaz.class.getCanonicalName()) - .setPath("/doubleField").setOrder(SortOrder.DESCENDING); + .setPath("/doubleField") + .setOrder(SortOrder.DESCENDING); String sortingColumn5 = EbeanLocalDAO.getSortingColumn(indexSortCriterion5); assertEquals(sortingColumn5, EbeanMetadataIndex.DOUBLE_COLUMN); // 6. float corresponds to double column IndexSortCriterion indexSortCriterion6 = new IndexSortCriterion().setAspect(AspectBaz.class.getCanonicalName()) - .setPath("/floatField").setOrder(SortOrder.DESCENDING); + .setPath("/floatField") + .setOrder(SortOrder.DESCENDING); String sortingColumn6 = EbeanLocalDAO.getSortingColumn(indexSortCriterion6); assertEquals(sortingColumn6, EbeanMetadataIndex.DOUBLE_COLUMN); // 7. enum corresponds to string column IndexSortCriterion indexSortCriterion7 = new IndexSortCriterion().setAspect(AspectBaz.class.getCanonicalName()) - .setPath("/enumField").setOrder(SortOrder.DESCENDING); + .setPath("/enumField") + .setOrder(SortOrder.DESCENDING); String sortingColumn7 = EbeanLocalDAO.getSortingColumn(indexSortCriterion7); assertEquals(sortingColumn7, EbeanMetadataIndex.STRING_COLUMN); // 8. nested field IndexSortCriterion indexSortCriterion8 = new IndexSortCriterion().setAspect(AspectBaz.class.getCanonicalName()) - .setPath("/recordField/value").setOrder(SortOrder.DESCENDING); + .setPath("/recordField/value") + .setOrder(SortOrder.DESCENDING); String sortingColumn8 = EbeanLocalDAO.getSortingColumn(indexSortCriterion8); assertEquals(sortingColumn8, EbeanMetadataIndex.STRING_COLUMN); // 9. invalid type IndexSortCriterion indexSortCriterion9 = new IndexSortCriterion().setAspect(AspectBaz.class.getCanonicalName()) - .setPath("/arrayField").setOrder(SortOrder.DESCENDING); - assertThrows(UnsupportedOperationException.class, - () -> EbeanLocalDAO.getSortingColumn(indexSortCriterion9)); + .setPath("/arrayField") + .setOrder(SortOrder.DESCENDING); + assertThrows(UnsupportedOperationException.class, () -> EbeanLocalDAO.getSortingColumn(indexSortCriterion9)); // 10. array of records is invalid type IndexSortCriterion indexSortCriterion10 = new IndexSortCriterion().setAspect(MixedRecord.class.getCanonicalName()) - .setPath("/recordArray/*/value").setOrder(SortOrder.DESCENDING); - assertThrows(UnsupportedOperationException.class, - () -> EbeanLocalDAO.getSortingColumn(indexSortCriterion10)); + .setPath("/recordArray/*/value") + .setOrder(SortOrder.DESCENDING); + assertThrows(UnsupportedOperationException.class, () -> EbeanLocalDAO.getSortingColumn(indexSortCriterion10)); } @Test @@ -947,7 +952,8 @@ public void testSorting() { IndexValue indexValue1 = new IndexValue(); indexValue1.setString("val"); IndexCriterion criterion1 = new IndexCriterion().setAspect(aspect1) - .setPathParams(new IndexPathParams().setPath("/value").setValue(indexValue1).setCondition(Condition.START_WITH)); + .setPathParams( + new IndexPathParams().setPath("/value").setValue(indexValue1).setCondition(Condition.START_WITH)); IndexCriterionArray indexCriterionArray1 = new IndexCriterionArray(Collections.singletonList(criterion1)); final IndexFilter indexFilter1 = new IndexFilter().setCriteria(indexCriterionArray1); @@ -956,15 +962,15 @@ public void testSorting() { assertEquals(urns1, Arrays.asList(urn1, urn2, urn3)); // filter and sort on same aspect and path - IndexSortCriterion indexSortCriterion1 = new IndexSortCriterion().setAspect(aspect1) - .setPath("/value").setOrder(SortOrder.DESCENDING); + IndexSortCriterion indexSortCriterion1 = + new IndexSortCriterion().setAspect(aspect1).setPath("/value").setOrder(SortOrder.DESCENDING); List urns2 = dao.listUrns(indexFilter1, indexSortCriterion1, null, 5); - assertEquals(urns2, Arrays.asList(urn2, urn1, urn3)); + assertEquals(urns2, Arrays.asList(urn2, urn1, urn3)); // filter and sort on different aspect and path - IndexSortCriterion indexSortCriterion2 = new IndexSortCriterion().setAspect(aspect2) - .setPath("/stringField").setOrder(SortOrder.ASCENDING); + IndexSortCriterion indexSortCriterion2 = + new IndexSortCriterion().setAspect(aspect2).setPath("/stringField").setOrder(SortOrder.ASCENDING); List urns3 = dao.listUrns(indexFilter1, indexSortCriterion2, null, 5); assertEquals(urns3, Arrays.asList(urn3, urn1, urn2)); @@ -973,20 +979,21 @@ public void testSorting() { IndexValue indexValue2 = new IndexValue(); indexValue2.setString("do"); IndexCriterion criterion2 = new IndexCriterion().setAspect(aspect2) - .setPathParams(new IndexPathParams().setPath("/stringField").setValue(indexValue2).setCondition(Condition.START_WITH)); + .setPathParams( + new IndexPathParams().setPath("/stringField").setValue(indexValue2).setCondition(Condition.START_WITH)); IndexCriterionArray indexCriterionArray2 = new IndexCriterionArray(Collections.singletonList(criterion2)); final IndexFilter indexFilter2 = new IndexFilter().setCriteria(indexCriterionArray2); - IndexSortCriterion indexSortCriterion3 = new IndexSortCriterion().setAspect(aspect2) - .setPath("/longField").setOrder(SortOrder.DESCENDING); + IndexSortCriterion indexSortCriterion3 = + new IndexSortCriterion().setAspect(aspect2).setPath("/longField").setOrder(SortOrder.DESCENDING); List urns4 = dao.listUrns(indexFilter2, indexSortCriterion3, null, 5); assertEquals(urns4, Arrays.asList(urn3, urn1)); // sorting on nested field - IndexSortCriterion indexSortCriterion4 = new IndexSortCriterion().setAspect(aspect2) - .setPath("/recordField/value").setOrder(SortOrder.ASCENDING); + IndexSortCriterion indexSortCriterion4 = + new IndexSortCriterion().setAspect(aspect2).setPath("/recordField/value").setOrder(SortOrder.ASCENDING); List urns5 = dao.listUrns(indexFilter1, indexSortCriterion4, null, 5); assertEquals(urns5, Arrays.asList(urn3, urn2, urn1)); @@ -1049,6 +1056,85 @@ public void testIn() { assertThrows(IllegalArgumentException.class, () -> dao.listUrns(indexFilter4, null, 5)); } + @Test + public void testCheckValidIndexCriterionArray() { + EbeanLocalDAO dao = createDao(FooUrn.class); + dao.enableLocalSecondaryIndex(true); + + // empty index criterion array + final IndexCriterionArray indexCriterionArray1 = new IndexCriterionArray(); + assertThrows(UnsupportedOperationException.class, () -> dao.checkValidIndexCriterionArray(indexCriterionArray1)); + + // >10 criterion in array + IndexValue indexValue = new IndexValue(); + indexValue.setString("val"); + IndexCriterion criterion = new IndexCriterion().setAspect(AspectFoo.class.getCanonicalName()) + .setPathParams(new IndexPathParams().setPath("/value").setValue(indexValue).setCondition(Condition.START_WITH)); + final IndexCriterionArray indexCriterionArray2 = new IndexCriterionArray(Collections.nCopies(11, criterion)); + assertThrows(UnsupportedOperationException.class, () -> dao.checkValidIndexCriterionArray(indexCriterionArray2)); + } + + @Test + public void testListUrnsOffsetPagination() { + EbeanLocalDAO dao = createDao(FooUrn.class); + dao.enableLocalSecondaryIndex(true); + FooUrn urn1 = makeFooUrn(1); + FooUrn urn2 = makeFooUrn(2); + FooUrn urn3 = makeFooUrn(3); + String aspect1 = AspectFoo.class.getCanonicalName(); + String aspect2 = AspectBaz.class.getCanonicalName(); + + addIndex(urn1, aspect1, "/value", "valB"); + addIndex(urn1, aspect2, "/stringField", "dolphin"); + addIndex(urn1, FooUrn.class.getCanonicalName(), "/fooId", 1); + + addIndex(urn2, aspect1, "/value", "valC"); + addIndex(urn2, aspect2, "/stringField", "reindeer"); + addIndex(urn2, FooUrn.class.getCanonicalName(), "/fooId", 2); + + addIndex(urn3, aspect1, "/value", "valA"); + addIndex(urn3, aspect2, "/stringField", "dog"); + addIndex(urn3, FooUrn.class.getCanonicalName(), "/fooId", 3); + + IndexValue indexValue = new IndexValue(); + indexValue.setString("val"); + IndexCriterion criterion = new IndexCriterion().setAspect(aspect1) + .setPathParams(new IndexPathParams().setPath("/value").setValue(indexValue).setCondition(Condition.START_WITH)); + + IndexCriterionArray indexCriterionArray = new IndexCriterionArray(Collections.singletonList(criterion)); + final IndexFilter indexFilter = new IndexFilter().setCriteria(indexCriterionArray); + + IndexSortCriterion indexSortCriterion = + new IndexSortCriterion().setAspect(aspect1).setPath("/value").setOrder(SortOrder.DESCENDING); + + // first page + ListResult results1 = dao.listUrns(indexFilter, indexSortCriterion, 0, 2); + assertEquals(results1.getValues(), Arrays.asList(urn2, urn1)); + assertTrue(results1.isHavingMore()); + assertEquals(results1.getNextStart(), 2); + assertEquals(results1.getTotalCount(), 3); + assertEquals(results1.getPageSize(), 2); + assertEquals(results1.getTotalPageCount(), 2); + + // last page + ListResult results2 = dao.listUrns(indexFilter, indexSortCriterion, 2, 2); + assertEquals(results2.getValues(), Arrays.asList(urn3)); + assertFalse(results2.isHavingMore()); + assertEquals(results2.getNextStart(), ListResult.INVALID_NEXT_START); + assertEquals(results2.getTotalCount(), 3); + assertEquals(results2.getPageSize(), 2); + assertEquals(results2.getTotalPageCount(), 2); + + // beyond last page + ListResult results3 = dao.listUrns(indexFilter, indexSortCriterion, 4, 2); + assertEquals(results3.getValues(), new ArrayList<>()); + assertFalse(results3.isHavingMore()); + assertEquals(results3.getNextStart(), ListResult.INVALID_NEXT_START); + assertEquals(results3.getTotalCount(), 3); + assertEquals(results3.getPageSize(), 2); + assertEquals(results3.getTotalPageCount(), 2); + } + @Test public void testListUrns() { EbeanLocalDAO dao = createDao(FooUrn.class); @@ -1139,6 +1225,16 @@ public void testGetAspectsWithIndexFilter() { UrnAspectEntry entry2 = new UrnAspectEntry<>(urn2, Arrays.asList(e2foo1, e2bar1)); assertEquals(actual, Arrays.asList(entry1, entry2)); + + // offset pagination + ListResult> actualListResult = dao.getAspects(aspectClasses, indexFilter, null, 0, 2); + + assertEquals(actualListResult.getValues(), Arrays.asList(entry1, entry2)); + assertFalse(actualListResult.isHavingMore()); + assertEquals(actualListResult.getNextStart(), ListResult.INVALID_NEXT_START); + assertEquals(actualListResult.getTotalCount(), 2); + assertEquals(actualListResult.getPageSize(), 2); + assertEquals(actualListResult.getTotalPageCount(), 1); } @Test @@ -1171,8 +1267,8 @@ public void testList() { assertNotNull(results.getMetadata()); List expectedVersions = Arrays.asList(0L, 1L, 2L, 3L, 4L); List expectedUrns = Arrays.asList(makeFooUrn(0), makeFooUrn(1), makeFooUrn(2), makeFooUrn(3), makeFooUrn(4)); - assertVersionMetadata(results.getMetadata(), expectedVersions, expectedUrns, 1234L, Urns.createFromTypeSpecificString("test", "foo"), - Urns.createFromTypeSpecificString("test", "bar")); + assertVersionMetadata(results.getMetadata(), expectedVersions, expectedUrns, 1234L, + Urns.createFromTypeSpecificString("test", "foo"), Urns.createFromTypeSpecificString("test", "bar")); // List next page results = dao.list(AspectFoo.class, urn0, 5, 9); @@ -1701,8 +1797,8 @@ void testUpdateArrayToLocalIndex() { AspectFoo aspectFoo5 = new AspectFoo().setValue("fooVal5"); AspectFoo aspectFoo6 = new AspectFoo().setValue("fooVal6"); - final AspectFooArray - aspectFooArray1 = new AspectFooArray(Arrays.asList(aspectFoo1, aspectFoo2, aspectFoo3, aspectFoo4)); + final AspectFooArray aspectFooArray1 = + new AspectFooArray(Arrays.asList(aspectFoo1, aspectFoo2, aspectFoo3, aspectFoo4)); final MixedRecord aspect1 = new MixedRecord().setRecordArray(aspectFooArray1); aspect1.setValue("val1"); @@ -2087,8 +2183,7 @@ public void testOptimisticLockException() { // call save method with timestamp 123 but timestamp is already changed to 456 // expect OptimisticLockException if optimistic locking is enabled - dao.saveWithOptimisticLocking(fooUrn, fooAspect, - makeAuditStamp("fooActor", 789), 0, false, new Timestamp(123)); + dao.saveWithOptimisticLocking(fooUrn, fooAspect, makeAuditStamp("fooActor", 789), 0, false, new Timestamp(123)); } private void addMetadata(Urn urn, String aspectName, long version, RecordTemplate metadata) {