From aa0fed68128794bf493a0d2018f8d44d5d349613 Mon Sep 17 00:00:00 2001 From: halprin Date: Wed, 6 Dec 2017 17:38:25 -0700 Subject: [PATCH 01/20] QPPCT-537: Changed Cpc and added the CpcProcessedCreateDate --- .../conversion/api/helper/MetadataHelper.java | 14 +++++- .../qpp/conversion/api/model/Metadata.java | 44 +++++++++++++++---- 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java index aa844e788..af1e2a71e 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java @@ -10,12 +10,15 @@ import java.util.Arrays; import java.util.List; import java.util.Objects; +import java.util.Random; /** * Utilities for working with Metadata beans */ public class MetadataHelper { + private static final Random RANDOM_HASH = new Random(); + private MetadataHelper() { } @@ -33,16 +36,23 @@ public static Metadata generateMetadata(Node node, Outcome outcome) { Metadata metadata = new Metadata(); - metadata.setApm(findApm(node)); + String apmId = findApm(node); + + metadata.setApm(apmId); metadata.setTin(findTin(node)); metadata.setNpi(findNpi(node)); - metadata.setCpc(isCpc(node)); + + metadata.setCpc(isCpc(node) ? cpcHash() : null); metadata.setCpcProcessed(false); outcome.setStatus(metadata); return metadata; } + private static String cpcHash() { + return "CPC_" + RANDOM_HASH.nextInt(32); + } + private static String findApm(Node node) { return findValue(node, ClinicalDocumentDecoder.ENTITY_ID, TemplateId.CLINICAL_DOCUMENT); } diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java index 302aafa3f..8d7564ca7 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java @@ -1,13 +1,14 @@ package gov.cms.qpp.conversion.api.model; import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBAttribute; -import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBAutoGenerateStrategy; import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBAutoGeneratedKey; -import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBAutoGeneratedTimestamp; import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBHashKey; import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTable; import com.amazonaws.services.dynamodbv2.datamodeling.encryption.DoNotEncrypt; import com.google.common.base.Objects; + +import java.time.Instant; +import java.time.format.DateTimeFormatter; import java.util.Date; /** @@ -18,7 +19,6 @@ public final class Metadata { private String uuid; private String tin; //this field is encrypted private String npi; - private Date createdDate; private String apm; private Long submissionYear; private String submissionLocator; @@ -27,12 +27,18 @@ public final class Metadata { private Boolean overallStatus; private Boolean conversionStatus; private Boolean validationStatus; - private Boolean cpc; + private String cpc; private String conversionErrorLocator; private String validationErrorLocator; private String rawValidationErrorLocator; + + private Date createdDate; private Boolean cpcProcessed; + public Metadata() { + createdDate = new Date(); + } + /** * The UUID that uniquely identifies this item. @@ -61,7 +67,7 @@ public void setUuid(String uuid) { */ @DoNotEncrypt @DynamoDBAttribute(attributeName = "CreateDate") - @DynamoDBAutoGeneratedTimestamp(strategy = DynamoDBAutoGenerateStrategy.CREATE) + //@DynamoDBAutoGeneratedTimestamp(strategy = DynamoDBAutoGenerateStrategy.CREATE) public Date getCreatedDate() { Date returnedDate = null; @@ -294,7 +300,7 @@ public void setValidationStatus(Boolean validationStatus) { */ @DoNotEncrypt @DynamoDBAttribute(attributeName = "Cpc") - public Boolean getCpc() { + public String getCpc() { return cpc; } @@ -303,7 +309,7 @@ public Boolean getCpc() { * * @param cpc A CPC+ conversion or not. */ - public void setCpc(Boolean cpc) { + public void setCpc(String cpc) { this.cpc = cpc; } @@ -378,8 +384,6 @@ public void setRawValidationErrorLocator(String rawValidationErrorLocator) { * * @return */ - @DoNotEncrypt - @DynamoDBAttribute(attributeName = "CpcProcessed") public Boolean getCpcProcessed() { return cpcProcessed; } @@ -393,6 +397,28 @@ public void setCpcProcessed(Boolean cpcProcessed) { this.cpcProcessed = cpcProcessed; } + @DoNotEncrypt + @DynamoDBAttribute(attributeName = "CpcProcessedCreateDate") + public String getCpcProcessedCreateDate() { + String combination = null; + + if (cpcProcessed != null) { + combination = cpcProcessed.toString() + "#" + DateTimeFormatter.ISO_INSTANT.format(createdDate.toInstant()); + } + + return combination; + } + + public void setCpcProcessedCreateDate(String combination) { + + String[] split = combination.split("#"); + String isProcessed = split[0]; + String creationDate = split[1]; + + setCpcProcessed(Boolean.valueOf(isProcessed)); + setCreatedDate(Date.from(Instant.parse(creationDate))); + } + /** * Determines the equality between this object and another. * From d5ad9f6b4fedc27b7bdefd031544a77b50ffeed4 Mon Sep 17 00:00:00 2001 From: halprin Date: Wed, 6 Dec 2017 18:25:01 -0700 Subject: [PATCH 02/20] QPPCT-537: toString for UnprocessedCpcFileData --- .../conversion/api/model/UnprocessedCpcFileData.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/UnprocessedCpcFileData.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/UnprocessedCpcFileData.java index d0f28e005..f78d3dcf0 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/UnprocessedCpcFileData.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/UnprocessedCpcFileData.java @@ -1,5 +1,7 @@ package gov.cms.qpp.conversion.api.model; +import com.google.common.base.MoreObjects; + import java.util.Date; /** @@ -70,4 +72,14 @@ public Boolean getValidationSuccess() { return validationSuccess; } + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("fileId", fileId) + .add("filename", "*could hold PII*") + .add("apm", apm) + .add("conversionDate", conversionDate) + .add("validationSuccess", validationSuccess) + .toString(); + } } From f0c72be0b20e3afe44457c7094a214d12e2486a5 Mon Sep 17 00:00:00 2001 From: halprin Date: Wed, 6 Dec 2017 18:29:13 -0700 Subject: [PATCH 03/20] QPPCT-537: Use a constant for the partition number for the CPC index --- .../java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java | 3 ++- .../main/java/gov/cms/qpp/conversion/api/model/Constants.java | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java index af1e2a71e..a468f0048 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java @@ -1,5 +1,6 @@ package gov.cms.qpp.conversion.api.helper; +import gov.cms.qpp.conversion.api.model.Constants; import gov.cms.qpp.conversion.api.model.Metadata; import gov.cms.qpp.conversion.decode.ClinicalDocumentDecoder; import gov.cms.qpp.conversion.decode.MultipleTinsDecoder; @@ -50,7 +51,7 @@ public static Metadata generateMetadata(Node node, Outcome outcome) { } private static String cpcHash() { - return "CPC_" + RANDOM_HASH.nextInt(32); + return "CPC_" + RANDOM_HASH.nextInt(Constants.CPC_DYNAMO_PARTITIONS); } private static String findApm(Node node) { diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Constants.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Constants.java index f9eabaa0c..369f59418 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Constants.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Constants.java @@ -13,6 +13,7 @@ public class Constants { public static final String VALIDATION_URL_ENV_VARIABLE = "VALIDATION_URL"; public static final String USE_SYNC_EXECUTOR = "USE_SYNC_EXECUTOR"; public static final String V1_API_ACCEPT = "application/vnd.qpp.cms.gov.v1+json"; + public static final Integer CPC_DYNAMO_PARTITIONS = 32; /** * Library utility class so the constructor is private and empty. From 3e8ed8ef850988d63cce13c228926f8485ee792f Mon Sep 17 00:00:00 2001 From: halprin Date: Wed, 6 Dec 2017 18:57:37 -0700 Subject: [PATCH 04/20] QPPCT-537: Iteratively query the index effeciently --- .../controllers/v1/CpcFileControllerV1.java | 5 +-- .../api/services/DbServiceImpl.java | 31 ++++++++++++------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/controllers/v1/CpcFileControllerV1.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/controllers/v1/CpcFileControllerV1.java index d60308f00..60270b724 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/controllers/v1/CpcFileControllerV1.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/controllers/v1/CpcFileControllerV1.java @@ -3,8 +3,6 @@ import gov.cms.qpp.conversion.api.model.Constants; import gov.cms.qpp.conversion.api.model.UnprocessedCpcFileData; import gov.cms.qpp.conversion.api.services.CpcFileService; -import java.io.IOException; -import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -14,6 +12,9 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; +import java.io.IOException; +import java.util.List; + /** * Controller to handle cpc file data */ diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java index c87d4f8ce..7f8d14a5e 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java @@ -1,19 +1,23 @@ package gov.cms.qpp.conversion.api.services; import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBMapper; -import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBScanExpression; +import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBQueryExpression; import com.amazonaws.services.dynamodbv2.model.AttributeValue; import gov.cms.qpp.conversion.api.model.Constants; import gov.cms.qpp.conversion.api.model.Metadata; -import java.util.HashMap; -import java.util.List; -import java.util.concurrent.CompletableFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.stereotype.Service; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + /** * Writes a {@link Metadata} object to DynamoDB. */ @@ -59,15 +63,20 @@ public CompletableFuture write(Metadata meta) { * @return {@link List} of unprocessed {@link Metadata} */ public List getUnprocessedCpcPlusMetaData() { - HashMap valueMap = new HashMap<>(); - valueMap.put(":cpcValue", new AttributeValue().withN("1")); - valueMap.put(":cpcProcessedValue", new AttributeValue().withN("0")); - DynamoDBScanExpression metadataScan = new DynamoDBScanExpression() - .withFilterExpression("Cpc = :cpcValue and CpcProcessed = :cpcProcessedValue") - .withExpressionAttributeValues(valueMap); + return IntStream.range(0, Constants.CPC_DYNAMO_PARTITIONS).mapToObj(partition -> { + HashMap valueMap = new HashMap<>(); + valueMap.put(":cpcValue", new AttributeValue().withS("CPC_" + partition)); + valueMap.put(":cpcProcessedValue", new AttributeValue().withS("false")); + + DynamoDBQueryExpression metadataQuery = new DynamoDBQueryExpression() + .withIndexName("Cpc-CpcProcessedCreateDate-index") + .withKeyConditionExpression("Cpc = :cpcValue and begins_with(CpcProcessedCreateDate, :cpcProcessedValue)") + .withExpressionAttributeValues(valueMap) + .withConsistentRead(false); - return mapper.scan(Metadata.class, metadataScan); + return mapper.query(Metadata.class, metadataQuery).stream(); + }).flatMap(Function.identity()).collect(Collectors.toList()); } /** From 8fd2b5fd6e688adec57611d0ca341df71c799a31 Mon Sep 17 00:00:00 2001 From: halprin Date: Wed, 6 Dec 2017 19:03:00 -0700 Subject: [PATCH 05/20] QPPCT-537: Create constant for start of CPC partition --- .../java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java | 2 +- .../main/java/gov/cms/qpp/conversion/api/model/Constants.java | 1 + .../java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java index a468f0048..550bc0ae1 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java @@ -51,7 +51,7 @@ public static Metadata generateMetadata(Node node, Outcome outcome) { } private static String cpcHash() { - return "CPC_" + RANDOM_HASH.nextInt(Constants.CPC_DYNAMO_PARTITIONS); + return Constants.CPC_DYNAMO_PARTITION_START + RANDOM_HASH.nextInt(Constants.CPC_DYNAMO_PARTITIONS); } private static String findApm(Node node) { diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Constants.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Constants.java index 369f59418..10b93cab6 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Constants.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Constants.java @@ -14,6 +14,7 @@ public class Constants { public static final String USE_SYNC_EXECUTOR = "USE_SYNC_EXECUTOR"; public static final String V1_API_ACCEPT = "application/vnd.qpp.cms.gov.v1+json"; public static final Integer CPC_DYNAMO_PARTITIONS = 32; + public static final String CPC_DYNAMO_PARTITION_START = "CPC_"; /** * Library utility class so the constructor is private and empty. diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java index 7f8d14a5e..e136ff95d 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java @@ -66,7 +66,7 @@ public List getUnprocessedCpcPlusMetaData() { return IntStream.range(0, Constants.CPC_DYNAMO_PARTITIONS).mapToObj(partition -> { HashMap valueMap = new HashMap<>(); - valueMap.put(":cpcValue", new AttributeValue().withS("CPC_" + partition)); + valueMap.put(":cpcValue", new AttributeValue().withS(Constants.CPC_DYNAMO_PARTITION_START + partition)); valueMap.put(":cpcProcessedValue", new AttributeValue().withS("false")); DynamoDBQueryExpression metadataQuery = new DynamoDBQueryExpression() From 5aac3e575f4b93cece35b0a3c3985870d9b565cf Mon Sep 17 00:00:00 2001 From: halprin Date: Wed, 6 Dec 2017 19:04:43 -0700 Subject: [PATCH 06/20] QPPCT-537: Delete commented out code --- .../src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java | 1 - 1 file changed, 1 deletion(-) diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java index 8d7564ca7..eb07f259c 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java @@ -67,7 +67,6 @@ public void setUuid(String uuid) { */ @DoNotEncrypt @DynamoDBAttribute(attributeName = "CreateDate") - //@DynamoDBAutoGeneratedTimestamp(strategy = DynamoDBAutoGenerateStrategy.CREATE) public Date getCreatedDate() { Date returnedDate = null; From 55728e8abe8cc901895b7d69671ed5fb93483adb Mon Sep 17 00:00:00 2001 From: halprin Date: Wed, 6 Dec 2017 20:17:49 -0700 Subject: [PATCH 07/20] QPPCT-537: Fixed-up tests --- .../qpp/conversion/api/model/Metadata.java | 5 +++++ .../api/helper/MetadataHelperTest.java | 9 ++++---- .../conversion/api/model/MetadataTest.java | 11 +++++++++- .../api/services/DbServiceImplTest.java | 22 +++++++++++-------- 4 files changed, 32 insertions(+), 15 deletions(-) diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java index eb07f259c..ea162ff77 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java @@ -411,6 +411,11 @@ public String getCpcProcessedCreateDate() { public void setCpcProcessedCreateDate(String combination) { String[] split = combination.split("#"); + + if (split.length < 2) { + return; + } + String isProcessed = split[0]; String creationDate = split[1]; diff --git a/rest-api/src/test/java/gov/cms/qpp/conversion/api/helper/MetadataHelperTest.java b/rest-api/src/test/java/gov/cms/qpp/conversion/api/helper/MetadataHelperTest.java index 7d1d774aa..e04f13acf 100644 --- a/rest-api/src/test/java/gov/cms/qpp/conversion/api/helper/MetadataHelperTest.java +++ b/rest-api/src/test/java/gov/cms/qpp/conversion/api/helper/MetadataHelperTest.java @@ -1,12 +1,11 @@ package gov.cms.qpp.conversion.api.helper; +import gov.cms.qpp.conversion.api.model.Constants; import gov.cms.qpp.conversion.api.model.Metadata; import gov.cms.qpp.conversion.decode.ClinicalDocumentDecoder; import gov.cms.qpp.conversion.decode.MultipleTinsDecoder; import gov.cms.qpp.conversion.model.Node; -import gov.cms.qpp.conversion.model.Program; import gov.cms.qpp.conversion.model.TemplateId; - import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -36,7 +35,7 @@ void testExtractsCpcProgramType() { node.putValue(ClinicalDocumentDecoder.PROGRAM_NAME, "CPCPLUS"); Metadata metadata = MetadataHelper.generateMetadata(node, MetadataHelper.Outcome.SUCCESS); - assertThat(metadata.getCpc()).isTrue(); + assertThat(metadata.getCpc()).startsWith(Constants.CPC_DYNAMO_PARTITION_START); } @Test @@ -48,7 +47,7 @@ void testExtractsCpcProgramTypeFromChild() { node.addChildNode(child); Metadata metadata = MetadataHelper.generateMetadata(node, MetadataHelper.Outcome.SUCCESS); - assertThat(metadata.getCpc()).isTrue(); + assertThat(metadata.getCpc()).startsWith(Constants.CPC_DYNAMO_PARTITION_START); } @Test @@ -59,7 +58,7 @@ void testChildLacksCpcPlus() { node.addChildNode(child); Metadata metadata = MetadataHelper.generateMetadata(node, MetadataHelper.Outcome.SUCCESS); - assertThat(metadata.getCpc()).isFalse(); + assertThat(metadata.getCpc()).isNull(); } @Test diff --git a/rest-api/src/test/java/gov/cms/qpp/conversion/api/model/MetadataTest.java b/rest-api/src/test/java/gov/cms/qpp/conversion/api/model/MetadataTest.java index 410ffe4b4..e03143e36 100644 --- a/rest-api/src/test/java/gov/cms/qpp/conversion/api/model/MetadataTest.java +++ b/rest-api/src/test/java/gov/cms/qpp/conversion/api/model/MetadataTest.java @@ -53,6 +53,15 @@ private boolean junk(Method method) { @SuppressWarnings("unchecked") private static T getDefaultValue(Class clazz) { - return (T) Array.get(Array.newInstance(clazz, 1), 0); + T defaultValue = null; + + try { + defaultValue = clazz.getConstructor().newInstance(); + } + catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + defaultValue = (T) Array.get(Array.newInstance(clazz, 1), 0); + } + + return defaultValue; } } diff --git a/rest-api/src/test/java/gov/cms/qpp/conversion/api/services/DbServiceImplTest.java b/rest-api/src/test/java/gov/cms/qpp/conversion/api/services/DbServiceImplTest.java index aa89dfdeb..4f4699d18 100644 --- a/rest-api/src/test/java/gov/cms/qpp/conversion/api/services/DbServiceImplTest.java +++ b/rest-api/src/test/java/gov/cms/qpp/conversion/api/services/DbServiceImplTest.java @@ -2,16 +2,11 @@ import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBMapper; -import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBScanExpression; -import com.amazonaws.services.dynamodbv2.datamodeling.PaginatedScanList; +import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBQueryExpression; +import com.amazonaws.services.dynamodbv2.datamodeling.PaginatedQueryList; import gov.cms.qpp.conversion.api.model.Constants; import gov.cms.qpp.conversion.api.model.Metadata; import gov.cms.qpp.test.MockitoExtension; -import java.lang.reflect.Array; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.concurrent.CompletableFuture; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -20,6 +15,10 @@ import org.springframework.core.env.Environment; import org.springframework.core.task.TaskExecutor; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Stream; + import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertWithMessage; import static org.mockito.ArgumentMatchers.any; @@ -87,14 +86,19 @@ void testNoWriteBecauseNoAudit() { verifyZeroInteractions(dbMapper); assertWithMessage("The returned metadata must be an empty metadata.") - .that(metadataOut).isEqualTo(new Metadata()); + .that(metadataOut.getUuid()).isNull(); } @Test void testGetUnprocessedCpcPlusMetaData() { + PaginatedQueryList mockMetadataList = mock(PaginatedQueryList.class); + when(mockMetadataList.stream()).thenAnswer(invocationOnMock -> Stream.of(new Metadata(), new Metadata())); + when(dbMapper.query(eq(Metadata.class), any(DynamoDBQueryExpression.class))).thenReturn(mockMetadataList); + List metaDataList = underTest.getUnprocessedCpcPlusMetaData(); - verify(dbMapper, times(1)).scan(any(Class.class), any(DynamoDBScanExpression.class)); + verify(dbMapper, times(Constants.CPC_DYNAMO_PARTITIONS)).query(any(Class.class), any(DynamoDBQueryExpression.class)); + assertThat(metaDataList).hasSize(2 * Constants.CPC_DYNAMO_PARTITIONS); } private Metadata writeMeta() { From 9d439872afe083fe82309e1a042e3aa3dbd55c20 Mon Sep 17 00:00:00 2001 From: halprin Date: Fri, 8 Dec 2017 11:20:07 -0700 Subject: [PATCH 08/20] QPPCT-537: Minor improvements --- .../gov/cms/qpp/conversion/api/helper/MetadataHelper.java | 8 +++----- .../java/gov/cms/qpp/conversion/api/model/Metadata.java | 2 +- .../cms/qpp/conversion/api/services/DbServiceImpl.java | 3 ++- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java index 550bc0ae1..1cd3ea667 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java @@ -11,14 +11,14 @@ import java.util.Arrays; import java.util.List; import java.util.Objects; -import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; /** * Utilities for working with Metadata beans */ public class MetadataHelper { - private static final Random RANDOM_HASH = new Random(); + private static final ThreadLocalRandom RANDOM_HASH = ThreadLocalRandom.current(); private MetadataHelper() { } @@ -37,9 +37,7 @@ public static Metadata generateMetadata(Node node, Outcome outcome) { Metadata metadata = new Metadata(); - String apmId = findApm(node); - - metadata.setApm(apmId); + metadata.setApm(findApm(node)); metadata.setTin(findTin(node)); metadata.setNpi(findNpi(node)); diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java index ea162ff77..1c2c807ba 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java @@ -397,7 +397,7 @@ public void setCpcProcessed(Boolean cpcProcessed) { } @DoNotEncrypt - @DynamoDBAttribute(attributeName = "CpcProcessedCreateDate") + @DynamoDBAttribute(attributeName = "CpcProcessed_CreateDate") public String getCpcProcessedCreateDate() { String combination = null; diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java index e136ff95d..c48e42e9e 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java @@ -13,6 +13,7 @@ import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.function.Function; import java.util.stream.Collectors; @@ -65,7 +66,7 @@ public CompletableFuture write(Metadata meta) { public List getUnprocessedCpcPlusMetaData() { return IntStream.range(0, Constants.CPC_DYNAMO_PARTITIONS).mapToObj(partition -> { - HashMap valueMap = new HashMap<>(); + Map valueMap = new HashMap<>(); valueMap.put(":cpcValue", new AttributeValue().withS(Constants.CPC_DYNAMO_PARTITION_START + partition)); valueMap.put(":cpcProcessedValue", new AttributeValue().withS("false")); From 9e652bcceb1db2494eb47f40d8c03f21d81ef553 Mon Sep 17 00:00:00 2001 From: halprin Date: Fri, 8 Dec 2017 11:43:03 -0700 Subject: [PATCH 09/20] QPPCT-537: Ignore CpcProcessed variable --- .../main/java/gov/cms/qpp/conversion/api/model/Metadata.java | 2 ++ .../gov/cms/qpp/conversion/api/services/DbServiceImpl.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java index 1c2c807ba..17f7e1888 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java @@ -3,6 +3,7 @@ import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBAttribute; import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBAutoGeneratedKey; import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBHashKey; +import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBIgnore; import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTable; import com.amazonaws.services.dynamodbv2.datamodeling.encryption.DoNotEncrypt; import com.google.common.base.Objects; @@ -383,6 +384,7 @@ public void setRawValidationErrorLocator(String rawValidationErrorLocator) { * * @return */ + @DynamoDBIgnore public Boolean getCpcProcessed() { return cpcProcessed; } diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java index c48e42e9e..2a540c084 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java @@ -71,8 +71,8 @@ public List getUnprocessedCpcPlusMetaData() { valueMap.put(":cpcProcessedValue", new AttributeValue().withS("false")); DynamoDBQueryExpression metadataQuery = new DynamoDBQueryExpression() - .withIndexName("Cpc-CpcProcessedCreateDate-index") - .withKeyConditionExpression("Cpc = :cpcValue and begins_with(CpcProcessedCreateDate, :cpcProcessedValue)") + .withIndexName("Cpc-CpcProcessed_CreateDate-index") + .withKeyConditionExpression("Cpc = :cpcValue and begins_with(CpcProcessed_CreateDate, :cpcProcessedValue)") .withExpressionAttributeValues(valueMap) .withConsistentRead(false); From f4cd974f1fa2022373f6aec171a02926f859f384 Mon Sep 17 00:00:00 2001 From: halprin Date: Fri, 8 Dec 2017 11:57:37 -0700 Subject: [PATCH 10/20] QPPCT-537: Use constants --- .../main/java/gov/cms/qpp/conversion/api/model/Constants.java | 2 ++ .../main/java/gov/cms/qpp/conversion/api/model/Metadata.java | 4 ++-- .../gov/cms/qpp/conversion/api/services/DbServiceImpl.java | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Constants.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Constants.java index 10b93cab6..467ec978a 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Constants.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Constants.java @@ -15,6 +15,8 @@ public class Constants { public static final String V1_API_ACCEPT = "application/vnd.qpp.cms.gov.v1+json"; public static final Integer CPC_DYNAMO_PARTITIONS = 32; public static final String CPC_DYNAMO_PARTITION_START = "CPC_"; + public static final String DYNAMO_CPC_ATTRIBUTE = "Cpc"; + public static final String DYNAMO_CPC_PROCESSED_CREATE_DATE_ATTRIBUTE = "CpcProcessed_CreateDate"; /** * Library utility class so the constructor is private and empty. diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java index 17f7e1888..16b14aa94 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java @@ -299,7 +299,7 @@ public void setValidationStatus(Boolean validationStatus) { * @return True for a CPC+ conversion, false otherwise. */ @DoNotEncrypt - @DynamoDBAttribute(attributeName = "Cpc") + @DynamoDBAttribute(attributeName = Constants.DYNAMO_CPC_ATTRIBUTE) public String getCpc() { return cpc; } @@ -399,7 +399,7 @@ public void setCpcProcessed(Boolean cpcProcessed) { } @DoNotEncrypt - @DynamoDBAttribute(attributeName = "CpcProcessed_CreateDate") + @DynamoDBAttribute(attributeName = Constants.DYNAMO_CPC_PROCESSED_CREATE_DATE_ATTRIBUTE) public String getCpcProcessedCreateDate() { String combination = null; diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java index 2a540c084..d70366fbf 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java @@ -72,7 +72,8 @@ public List getUnprocessedCpcPlusMetaData() { DynamoDBQueryExpression metadataQuery = new DynamoDBQueryExpression() .withIndexName("Cpc-CpcProcessed_CreateDate-index") - .withKeyConditionExpression("Cpc = :cpcValue and begins_with(CpcProcessed_CreateDate, :cpcProcessedValue)") + .withKeyConditionExpression(Constants.DYNAMO_CPC_ATTRIBUTE + " = :cpcValue and begins_with(" + + Constants.DYNAMO_CPC_PROCESSED_CREATE_DATE_ATTRIBUTE + ", :cpcProcessedValue)") .withExpressionAttributeValues(valueMap) .withConsistentRead(false); From 3d50c0bbef8c3c8e4d6af045ba47bc92b58cae56 Mon Sep 17 00:00:00 2001 From: halprin Date: Fri, 8 Dec 2017 12:20:55 -0700 Subject: [PATCH 11/20] QPPCT-537: More tests for Metadata --- .../conversion/api/model/MetadataTest.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/rest-api/src/test/java/gov/cms/qpp/conversion/api/model/MetadataTest.java b/rest-api/src/test/java/gov/cms/qpp/conversion/api/model/MetadataTest.java index e03143e36..2ed8c9c4c 100644 --- a/rest-api/src/test/java/gov/cms/qpp/conversion/api/model/MetadataTest.java +++ b/rest-api/src/test/java/gov/cms/qpp/conversion/api/model/MetadataTest.java @@ -9,8 +9,10 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Arrays; +import java.util.Date; import java.util.function.Consumer; +import static com.google.common.truth.Truth.assertThat; import static junit.framework.TestCase.fail; class MetadataTest { @@ -23,6 +25,50 @@ void equalsContract() { .verify(); } + @Test + void testDateCreatedOnConstruction() { + Metadata metadata = new Metadata(); + assertThat(metadata.getCreatedDate()).isNotNull(); + } + + @Test + void testGetCpcProcessedCreateDateWithNullProcessed() { + Metadata metadata = new Metadata(); + assertThat(metadata.getCpcProcessedCreateDate()).isNull(); + } + + @Test + void testGetCpcProcessedCreateDateWithNonNullProcessed() { + Metadata metadata = new Metadata(); + Boolean processed = false; + metadata.setCpcProcessed(processed); + assertThat(metadata.getCpcProcessedCreateDate()).startsWith(processed + "#"); + } + + @Test + void testSetCpcProcessedCreateDateWithoutHash() { + Metadata metadata = new Metadata(); + Boolean processedBefore = metadata.getCpcProcessed(); + Date createDateBefore = metadata.getCreatedDate(); + + metadata.setCpcProcessedCreateDate("DogCow"); + + assertThat(metadata.getCpcProcessed()).isEqualTo(processedBefore); + assertThat(metadata.getCreatedDate()).isEqualTo(createDateBefore); + } + + @Test + void testSetCpcProcessedCreateDateWithHash() { + Metadata metadata = new Metadata(); + metadata.setCpcProcessed(false); + Date createDateBefore = metadata.getCreatedDate(); + + metadata.setCpcProcessedCreateDate("true#2017-12-08T18:32:54.846Z"); + + assertThat(metadata.getCpcProcessed()).isTrue(); + assertThat(metadata.getCreatedDate()).isLessThan(createDateBefore); + } + @Test void plumbing() { Consumer consumer = harness(new Metadata()); From f0f601d732a0fb10ed80a706775e0ebe19209386 Mon Sep 17 00:00:00 2001 From: Samuel Aquino Date: Fri, 8 Dec 2017 13:30:56 -0600 Subject: [PATCH 12/20] Javadocs for MetadataHelper --- .../conversion/api/helper/MetadataHelper.java | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java index 1cd3ea667..2a0be1ab8 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java @@ -18,9 +18,13 @@ */ public class MetadataHelper { - private static final ThreadLocalRandom RANDOM_HASH = ThreadLocalRandom.current(); + private static final ThreadLocalRandom RANDOM_HASH = ThreadLocalRandom.current(); + /** + * No need for constructor in this utility class + */ private MetadataHelper() { + //empty } /** @@ -48,24 +52,49 @@ public static Metadata generateMetadata(Node node, Outcome outcome) { return metadata; } + /** + * Retrieves the random hash for CPC Field + * + * @return Cpc field randomly hashed + */ private static String cpcHash() { return Constants.CPC_DYNAMO_PARTITION_START + RANDOM_HASH.nextInt(Constants.CPC_DYNAMO_PARTITIONS); } + /** + * Retrieves the APM Entity Id from the given node + * + * @return Apm Entity ID value + */ private static String findApm(Node node) { return findValue(node, ClinicalDocumentDecoder.ENTITY_ID, TemplateId.CLINICAL_DOCUMENT); } + /** + * Retrieves the Taxpayer Identification Number from the given node + * + * @return TIN value + */ private static String findTin(Node node) { return findValue(node, MultipleTinsDecoder.TAX_PAYER_IDENTIFICATION_NUMBER, TemplateId.QRDA_CATEGORY_III_REPORT_V3, TemplateId.CLINICAL_DOCUMENT); } + /** + * Retrieves the National Provider Identifier from the given node + * + * @return NPI value + */ private static String findNpi(Node node) { return findValue(node, MultipleTinsDecoder.NATIONAL_PROVIDER_IDENTIFIER, TemplateId.QRDA_CATEGORY_III_REPORT_V3, TemplateId.CLINICAL_DOCUMENT); } + /** + * Retrieves the random hash for CPC Field + * + * @return Cpc field randomly hashed + */ private static boolean isCpc(Node node) { if (Program.isCpc(node)) { return true; @@ -95,6 +124,15 @@ private static String findValue(Node node, String key, TemplateId... possibleLoc return found == null ? null : found.getValue(key); } + /** + * Finds all possible children within the given node for each {@link TemplateId} given + * filtered by children with specific keys + * + * @param node Object to search through + * @param key value to filter + * @param possibleLocations areas which the child can exist + * @return A child node with the correct value or null + */ private static Node findPossibleChildNode(Node node, String key, TemplateId... possibleLocations) { return Arrays.stream(possibleLocations) .distinct() From 23fd351da147b23d3baadbea6d9188f468da0479 Mon Sep 17 00:00:00 2001 From: halprin Date: Fri, 8 Dec 2017 12:37:10 -0700 Subject: [PATCH 13/20] QPPCT-537: More JavaDocs for Metadata --- .../qpp/conversion/api/model/Metadata.java | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java index 16b14aa94..b083ef24b 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java @@ -36,6 +36,9 @@ public final class Metadata { private Date createdDate; private Boolean cpcProcessed; + /** + * Constructs a new {@code Metadata} with the {@code createdDate} filled in upon construction. + */ public Metadata() { createdDate = new Date(); } @@ -382,7 +385,9 @@ public void setRawValidationErrorLocator(String rawValidationErrorLocator) { /** * Whether the file was processed by the CPC+ team * - * @return + * Ignored when writing to DynamoDB because {@code CpcProcessed_CreateDate} holds the pertinent information. + * + * @return Whether the file was processed. */ @DynamoDBIgnore public Boolean getCpcProcessed() { @@ -398,6 +403,13 @@ public void setCpcProcessed(Boolean cpcProcessed) { this.cpcProcessed = cpcProcessed; } + /** + * Returns an attribute that combines the CPC+ processed state and the date of creation. + * + * This is mostly useful in the CPC+ global secondary index. + * + * @return The combined attribute. + */ @DoNotEncrypt @DynamoDBAttribute(attributeName = Constants.DYNAMO_CPC_PROCESSED_CREATE_DATE_ATTRIBUTE) public String getCpcProcessedCreateDate() { @@ -410,6 +422,15 @@ public String getCpcProcessedCreateDate() { return combination; } + /** + * Sets the separate CPC+ processed flag and created date based on the argument + * + * Splits the the processed flag from the date by a {@code #} character. + * The first field must be {@code true} or {@code false} which represents the CPC+ processed boolean. + * The second field must be an ISO 8601 timestamp string. For example, {@code 2017-12-08T18:32:54.846Z}. + * + * @param combination The combined attribute. + */ public void setCpcProcessedCreateDate(String combination) { String[] split = combination.split("#"); From 6e0e74f27fe4ee2c1456ef3247167bcc23deff27 Mon Sep 17 00:00:00 2001 From: Samuel Aquino Date: Fri, 8 Dec 2017 13:46:42 -0600 Subject: [PATCH 14/20] Javadocs for AuditService --- .../conversion/api/services/AuditService.java | 4 +++ .../api/services/AuditServiceImpl.java | 28 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/AuditService.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/AuditService.java index 90024c994..2217832f9 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/AuditService.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/AuditService.java @@ -3,8 +3,12 @@ import gov.cms.qpp.conversion.Converter.ConversionReport; +import gov.cms.qpp.conversion.api.model.Metadata; import java.util.concurrent.CompletableFuture; +/** + * Service Interface for auditing {@link Metadata} by {@link ConversionReport} + */ public interface AuditService { /** * Report a successful conversion. diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/AuditServiceImpl.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/AuditServiceImpl.java index cf87f6de5..d9eebf9dc 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/AuditServiceImpl.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/AuditServiceImpl.java @@ -17,6 +17,9 @@ import java.util.UUID; import java.util.concurrent.CompletableFuture; +/** + * Service for storing {@link Metadata} by {@link Converter.ConversionReport} outcome + */ @Service public class AuditServiceImpl implements AuditService { private static final Logger API_LOG = LoggerFactory.getLogger(Constants.API_LOG); @@ -95,6 +98,11 @@ public CompletableFuture failValidation(Converter.ConversionReport convers return allWrites.whenComplete((nada, thrown) -> persist(metadata, thrown)); } + /** + * Determines if the No Audit Environment variable was passed + * + * @return the status of auditing + */ private boolean noAudit() { String noAudit = environment.getProperty(Constants.NO_AUDIT_ENV_VARIABLE); boolean returnValue = noAudit != null && !noAudit.isEmpty(); @@ -106,17 +114,37 @@ private boolean noAudit() { return returnValue; } + /** + * Initializes {@link Metadata} from the {@link Converter.ConversionReport} and conversion outcome + * + * @param report Object containing metadata information + * @param outcome Status of the conversion + * @return Constructed metadata + */ private Metadata initMetadata(Converter.ConversionReport report, MetadataHelper.Outcome outcome) { Metadata metadata = MetadataHelper.generateMetadata(report.getDecoded(), outcome); metadata.setFileName(report.getFilename()); return metadata; } + /** + * Calls the service to store the given file + * + * @param content to be stored + * @return the result of the storage upload + */ private CompletableFuture storeContent(InputStream content) { UUID key = UUID.randomUUID(); return storageService.store(key.toString(), content); } + /** + * Calls the service for writing {@link Metadata} to the database + * + * @param metadata to be stored + * @param thrown check for {@link AuditException} + * @return A {@link CompletableFuture} that will hold the written Metadata. + */ private CompletableFuture persist(Metadata metadata, Throwable thrown) { if (thrown != null) { throw new AuditException(thrown); From 1d7b69c0ba134a3579f10b32306ba1ac990cbe32 Mon Sep 17 00:00:00 2001 From: halprin Date: Fri, 8 Dec 2017 16:30:53 -0700 Subject: [PATCH 15/20] QPPCT-537: Use a constant instead of a magic number --- .../java/gov/cms/qpp/conversion/api/model/Metadata.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java index b083ef24b..d1731bb29 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java @@ -17,6 +17,8 @@ */ @DynamoDBTable(tableName = "ConversionMetadata") public final class Metadata { + private static final int CPC_PROCESSED_CREATE_DATE_NUM_FIELDS = 2; + private String uuid; private String tin; //this field is encrypted private String npi; @@ -435,12 +437,12 @@ public void setCpcProcessedCreateDate(String combination) { String[] split = combination.split("#"); - if (split.length < 2) { + if (split.length < CPC_PROCESSED_CREATE_DATE_NUM_FIELDS) { return; } - String isProcessed = split[0]; - String creationDate = split[1]; + String isProcessed = split[CPC_PROCESSED_CREATE_DATE_NUM_FIELDS - 2]; + String creationDate = split[CPC_PROCESSED_CREATE_DATE_NUM_FIELDS - 1]; setCpcProcessed(Boolean.valueOf(isProcessed)); setCreatedDate(Date.from(Instant.parse(creationDate))); From c8cb1db068484282d0a3f03163c60f5bf5d8e114 Mon Sep 17 00:00:00 2001 From: halprin Date: Thu, 14 Dec 2017 09:03:08 -0700 Subject: [PATCH 16/20] QPPCT-537: Updated JavaDocs --- .../java/gov/cms/qpp/conversion/api/model/Metadata.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java index 71f9cabd5..bc92e9235 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java @@ -301,7 +301,10 @@ public void setValidationStatus(Boolean validationStatus) { /** * Whether the conversion was for the CPC+ program. * - * @return True for a CPC+ conversion, false otherwise. + * This is set to a {@link String} that contains "CPC_" plus a number for DynamoDB partitioning of the GSI. + * If this method returns {@code null}, this was not a CPC+ conversion. + * + * @return A {@link String} for a CPC+ conversion, null otherwise. */ @DoNotEncrypt @DynamoDBAttribute(attributeName = Constants.DYNAMO_CPC_ATTRIBUTE) @@ -312,6 +315,9 @@ public String getCpc() { /** * Sets whether the conversion was for the CPC+ program. * + * If not {@code null}, must be of the form "CPC_" plus a number. + * Setting this to {@code null}, indicates this was not a CPC+ conversion. + * * @param cpc A CPC+ conversion or not. */ public void setCpc(String cpc) { From 15b5bfd838a4f42e3ba1a74a4a9897737cf90f22 Mon Sep 17 00:00:00 2001 From: halprin Date: Thu, 14 Dec 2017 09:21:50 -0700 Subject: [PATCH 17/20] QPPCT-537: Improve code readability --- .../conversion/api/helper/MetadataHelper.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java index af8ace96e..9df50d286 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/helper/MetadataHelper.java @@ -43,7 +43,7 @@ public static Metadata generateMetadata(Node node, Outcome outcome) { metadata.setApm(findApm(node)); metadata.setTin(findTin(node)); metadata.setNpi(findNpi(node)); - metadata.setCpc(isCpc(node) ? cpcHash() : null); + metadata.setCpc(deriveCpcHash(node)); metadata.setCpcProcessed(false); } @@ -53,12 +53,18 @@ public static Metadata generateMetadata(Node node, Outcome outcome) { } /** - * Retrieves the random hash for CPC Field + * Retrieves the random hash for the Cpc field if this is a CPC+ conversion. * - * @return Cpc field randomly hashed + * @return Cpc field randomly hashed or null if this isn't a CPC+ conversion */ - private static String cpcHash() { - return Constants.CPC_DYNAMO_PARTITION_START + RANDOM_HASH.nextInt(Constants.CPC_DYNAMO_PARTITIONS); + private static String deriveCpcHash(Node node) { + String cpcHash = null; + + if (isCpc(node)) { + cpcHash = Constants.CPC_DYNAMO_PARTITION_START + RANDOM_HASH.nextInt(Constants.CPC_DYNAMO_PARTITIONS); + } + + return cpcHash; } /** From eafaefbfe6327f9ccb07abb6747c004894b1fed6 Mon Sep 17 00:00:00 2001 From: halprin Date: Thu, 14 Dec 2017 09:31:29 -0700 Subject: [PATCH 18/20] QPPCT-537: More magic number removal --- .../java/gov/cms/qpp/conversion/api/model/Metadata.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java index bc92e9235..cbed0ba45 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/model/Metadata.java @@ -18,6 +18,8 @@ @DynamoDBTable(tableName = "ConversionMetadata") public final class Metadata { private static final int CPC_PROCESSED_CREATE_DATE_NUM_FIELDS = 2; + private static final int CPC_PROCESSED_INDEX = 0; + private static final int CPC_CREATE_DATE_INDEX = 1; private String uuid; private String tin; //this field is encrypted @@ -447,8 +449,8 @@ public void setCpcProcessedCreateDate(String combination) { return; } - String isProcessed = split[CPC_PROCESSED_CREATE_DATE_NUM_FIELDS - 2]; - String creationDate = split[CPC_PROCESSED_CREATE_DATE_NUM_FIELDS - 1]; + String isProcessed = split[CPC_PROCESSED_INDEX]; + String creationDate = split[CPC_CREATE_DATE_INDEX]; setCpcProcessed(Boolean.valueOf(isProcessed)); setCreatedDate(Date.from(Instant.parse(creationDate))); From b9f75f67c119f800401b51ba2d3c3e91134abc52 Mon Sep 17 00:00:00 2001 From: halprin Date: Mon, 18 Dec 2017 08:04:16 -0700 Subject: [PATCH 19/20] QPPCT-537: Equalize dates for Metadata in a test --- .../gov/cms/qpp/conversion/api/helper/MetadataHelperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rest-api/src/test/java/gov/cms/qpp/conversion/api/helper/MetadataHelperTest.java b/rest-api/src/test/java/gov/cms/qpp/conversion/api/helper/MetadataHelperTest.java index 24136af0c..a152347f7 100644 --- a/rest-api/src/test/java/gov/cms/qpp/conversion/api/helper/MetadataHelperTest.java +++ b/rest-api/src/test/java/gov/cms/qpp/conversion/api/helper/MetadataHelperTest.java @@ -25,6 +25,8 @@ void testGenerateMetadataForNullNodeReturnsSkinnyMetadata() { comparison.setValidationStatus(false); Metadata metadata = MetadataHelper.generateMetadata(null, outcome); + metadata.setCreatedDate(comparison.getCreatedDate()); + assertThat(metadata).isEqualTo(comparison); } From 6821d4858f21ed5fdd9445e4c0df31cc98e72cb3 Mon Sep 17 00:00:00 2001 From: halprin Date: Thu, 21 Dec 2017 15:57:25 -0700 Subject: [PATCH 20/20] QPPCT-537: Enhance CPC+ endpoint 2 --- .../api/services/CpcFileServiceImpl.java | 11 +++++------ .../api/services/CpcFileServiceImplTest.java | 15 ++++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/CpcFileServiceImpl.java b/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/CpcFileServiceImpl.java index 57ce882f3..b21ac0a5d 100644 --- a/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/CpcFileServiceImpl.java +++ b/rest-api/src/main/java/gov/cms/qpp/conversion/api/services/CpcFileServiceImpl.java @@ -3,13 +3,13 @@ import gov.cms.qpp.conversion.api.exceptions.NoFileInDatabaseException; import gov.cms.qpp.conversion.api.model.Metadata; import gov.cms.qpp.conversion.api.model.UnprocessedCpcFileData; -import java.io.IOException; -import java.util.List; -import java.util.stream.Collectors; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.InputStreamResource; import org.springframework.stereotype.Service; +import java.util.List; +import java.util.stream.Collectors; + /** * Service for handling Cpc File meta data */ @@ -41,11 +41,10 @@ public List getUnprocessedCpcPlusFiles() { * * @param fileId {@link Metadata} identifier * @return file contents as a {@link String} - * @throws IOException */ - public InputStreamResource getFileById(String fileId) throws IOException { + public InputStreamResource getFileById(String fileId) { Metadata metadata = dbService.getMetadataById(fileId); - if (metadata != null && metadata.getCpc() && !metadata.getCpcProcessed()) { + if (metadata != null && metadata.getCpc() != null && !metadata.getCpcProcessed()) { return new InputStreamResource(storageService.getFileByLocationId(metadata.getSubmissionLocator())); } else { throw new NoFileInDatabaseException(FILE_NOT_FOUND); diff --git a/rest-api/src/test/java/gov/cms/qpp/conversion/api/services/CpcFileServiceImplTest.java b/rest-api/src/test/java/gov/cms/qpp/conversion/api/services/CpcFileServiceImplTest.java index b7884a9cf..900542caf 100644 --- a/rest-api/src/test/java/gov/cms/qpp/conversion/api/services/CpcFileServiceImplTest.java +++ b/rest-api/src/test/java/gov/cms/qpp/conversion/api/services/CpcFileServiceImplTest.java @@ -3,12 +3,6 @@ import gov.cms.qpp.conversion.api.exceptions.NoFileInDatabaseException; import gov.cms.qpp.conversion.api.model.Metadata; import gov.cms.qpp.test.MockitoExtension; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -18,6 +12,13 @@ import org.mockito.Mock; import org.springframework.core.io.InputStreamResource; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import static com.google.common.truth.Truth.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.anyString; @@ -106,7 +107,7 @@ void testGetFileByIdNoFile() throws IOException { Metadata buildFakeMetadata(boolean isCpc, boolean isCpcProcessed) { Metadata metadata = new Metadata(); - metadata.setCpc(isCpc); + metadata.setCpc(isCpc ? "CPC_26" : null); metadata.setCpcProcessed(isCpcProcessed); metadata.setSubmissionLocator("test");