diff --git a/Jenkinsfile b/Jenkinsfile index 65ed513cd..44b0ae123 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,7 +6,7 @@ node('build-slave') { try { cleanWs() - stage('Checkout'){ + stage('Checkout'){ checkout scm } @@ -24,7 +24,7 @@ node('build-slave') { sh 'ls -al ~/' sh('chmod 777 ./dockerPushToRepo.sh') sh 'ARTIFACT_LABEL=bronze ./dockerPushToRepo.sh' - sh './metadata.sh > metadata.json' + sh './metadata.sh > metadata.json' sh 'cat metadata.json' archive includes: "metadata.json" } diff --git a/actors/core/src/main/java/org/sunbird/actor/core/BaseActor.java b/actors/core/src/main/java/org/sunbird/actor/core/BaseActor.java index 00c1efee4..475431ed7 100644 --- a/actors/core/src/main/java/org/sunbird/actor/core/BaseActor.java +++ b/actors/core/src/main/java/org/sunbird/actor/core/BaseActor.java @@ -22,6 +22,7 @@ public void onReceive(Object message) throws Throwable { if (message instanceof Request) { Request request = (Request) message; String callerName = request.getOperation(); + ProjectLogger.log("BaseActor onReceive called for operation : " + callerName); try { onReceive(request); } catch (Exception e) { diff --git a/actors/core/src/main/java/org/sunbird/actor/core/service/InterServiceCommunication.java b/actors/core/src/main/java/org/sunbird/actor/core/service/InterServiceCommunication.java new file mode 100644 index 000000000..580e1e308 --- /dev/null +++ b/actors/core/src/main/java/org/sunbird/actor/core/service/InterServiceCommunication.java @@ -0,0 +1,9 @@ +package org.sunbird.actor.core.service; + +import org.sunbird.common.request.Request; + +/** Created by arvind on 24/4/18. */ +public interface InterServiceCommunication { + + public Object getResponse(Request request, String operation); +} diff --git a/actors/core/src/main/java/org/sunbird/actor/core/service/InterServiceCommunicationFactory.java b/actors/core/src/main/java/org/sunbird/actor/core/service/InterServiceCommunicationFactory.java new file mode 100644 index 000000000..9bab1ca6b --- /dev/null +++ b/actors/core/src/main/java/org/sunbird/actor/core/service/InterServiceCommunicationFactory.java @@ -0,0 +1,47 @@ +package org.sunbird.actor.core.service; + +import java.util.HashMap; +import java.util.Map; +import org.sunbird.actor.core.service.impl.InterServiceCommunicationImpl; + +/** Created by arvind on 24/4/18. */ +public class InterServiceCommunicationFactory { + + private static InterServiceCommunicationFactory factory; + private static Map modes = new HashMap<>(); + + private InterServiceCommunicationFactory() {} + + public static InterServiceCommunicationFactory getInstance() { + if (null == factory) { + synchronized (InterServiceCommunicationFactory.class) { + if (null == factory) { + factory = new InterServiceCommunicationFactory(); + } + } + } + return factory; + } + + public InterServiceCommunication getCommunicationPath(String mode) { + if ("actorCommunication".equalsIgnoreCase(mode)) { + return getActorCommunicationMode(); + } + return null; + } + + public InterServiceCommunication getActorCommunicationMode() { + + if (modes.get("actorCommunication") != null) { + return modes.get("actorCommunication"); + } else { + synchronized (InterServiceCommunicationFactory.class) { + if (modes.get("actorCommunication") == null) { + InterServiceCommunication communication = new InterServiceCommunicationImpl(); + modes.put("actorCommunication", communication); + } + } + } + return modes.get("actorCommunication"); + } +} diff --git a/actors/core/src/main/java/org/sunbird/actor/core/service/impl/InterServiceCommunicationImpl.java b/actors/core/src/main/java/org/sunbird/actor/core/service/impl/InterServiceCommunicationImpl.java new file mode 100644 index 000000000..70c14741c --- /dev/null +++ b/actors/core/src/main/java/org/sunbird/actor/core/service/impl/InterServiceCommunicationImpl.java @@ -0,0 +1,59 @@ +package org.sunbird.actor.core.service.impl; + +import static akka.pattern.PatternsCS.ask; + +import akka.actor.ActorRef; +import akka.actor.ActorSelection; +import akka.util.Timeout; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.TimeUnit; +import org.sunbird.actor.core.service.InterServiceCommunication; +import org.sunbird.actor.router.RequestRouter; +import org.sunbird.actor.service.BaseMWService; +import org.sunbird.common.models.util.LoggerEnum; +import org.sunbird.common.models.util.ProjectLogger; +import org.sunbird.common.request.Request; +import scala.concurrent.duration.Duration; + +/** Created by arvind on 24/4/18. */ +public class InterServiceCommunicationImpl extends BaseMWService + implements InterServiceCommunication { + + private Integer WAIT_TIME = 10; + + @Override + public Object getResponse(Request request, String operation) { + ActorRef actor = RequestRouter.getActor(operation); + Timeout t = new Timeout(Duration.create(WAIT_TIME, TimeUnit.SECONDS)); + request.setOperation(operation); + Object obj = null; + if (null == actor) { + ActorSelection select = getRemoteRouter(RequestRouter.class.getSimpleName()); + CompletionStage futureActor = + select.resolveOneCS(Duration.create(WAIT_TIME, "seconds")); + try { + actor = futureActor.toCompletableFuture().get(); + } catch (Exception e) { + ProjectLogger.log( + "InterServiceCommunicationImpl : getResponse - unable to get actorref from actorselection " + + e.getMessage(), + e); + } + } + if (null == actor) { + ProjectLogger.log( + "InterServiceCommunicationImpl : getResponse - actorRef is null ", LoggerEnum.INFO); + return obj; + } + ProjectLogger.log( + "Operation " + operation + " with request type " + (request instanceof Request)); + CompletableFuture future = ask(actor, request, t).toCompletableFuture(); + try { + obj = future.get(WAIT_TIME + 2, TimeUnit.SECONDS); + } catch (Exception e) { + ProjectLogger.log("Interservice communication error " + e.getMessage(), e); + } + return obj; + } +} diff --git a/actors/core/src/main/java/org/sunbird/bean/Organization.java b/actors/core/src/main/java/org/sunbird/bean/Organization.java new file mode 100644 index 000000000..bd0a88245 --- /dev/null +++ b/actors/core/src/main/java/org/sunbird/bean/Organization.java @@ -0,0 +1,334 @@ +package org.sunbird.bean; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.Serializable; +import java.sql.Timestamp; +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(Include.NON_NULL) +public class Organization implements Serializable { + + private static final long serialVersionUID = 3617862727235741692L; + private String id; + private String addressId; + private String approvedBy; + private String approvedDate; + private String channel; + private String communityId; + private String contactDetail; + private String createdBy; + private String createdDate; + private Timestamp dateTime; + private String description; + private String externalId; + private String hashTagId; + private String homeUrl; + private String imgUrl; + private Boolean isApproved; + private Boolean isDefault; + private Boolean isRootOrg; + private String locationId; + private Integer noOfMembers; + private String orgCode; + private String orgName; + private String orgType; + private String orgTypeId; + private String parentOrgId; + private String preferredLanguage; + private String provider; + private String rootOrgId; + private String slug; + private Integer status; + private String theme; + private String thumbnail; + private String updatedBy; + private String updatedDate; + private List locationIds; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getAddressId() { + return addressId; + } + + public void setAddressId(String addressId) { + this.addressId = addressId; + } + + public String getApprovedBy() { + return approvedBy; + } + + public void setApprovedBy(String approvedBy) { + this.approvedBy = approvedBy; + } + + public String getApprovedDate() { + return approvedDate; + } + + public void setApprovedDate(String approvedDate) { + this.approvedDate = approvedDate; + } + + public String getChannel() { + return channel; + } + + public void setChannel(String channel) { + this.channel = channel; + } + + public String getCommunityId() { + return communityId; + } + + public void setCommunityId(String communityId) { + this.communityId = communityId; + } + + public String getContactDetail() { + return contactDetail; + } + + public void setContactDetail(String contactDetail) { + this.contactDetail = contactDetail; + } + + public String getCreatedBy() { + return createdBy; + } + + public void setCreatedBy(String createdBy) { + this.createdBy = createdBy; + } + + public String getCreatedDate() { + return createdDate; + } + + public void setCreatedDate(String createdDate) { + this.createdDate = createdDate; + } + + public Timestamp getDateTime() { + return dateTime; + } + + public void setDateTime(Timestamp dateTime) { + this.dateTime = dateTime; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getExternalId() { + return externalId; + } + + public void setExternalId(String externalId) { + this.externalId = externalId; + } + + public String getHashTagId() { + return hashTagId; + } + + public void setHashTagId(String hashTagId) { + this.hashTagId = hashTagId; + } + + public String getHomeUrl() { + return homeUrl; + } + + public void setHomeUrl(String homeUrl) { + this.homeUrl = homeUrl; + } + + public String getImgUrl() { + return imgUrl; + } + + public void setImgUrl(String imgUrl) { + this.imgUrl = imgUrl; + } + + public String getLocationId() { + return locationId; + } + + public void setLocationId(String locationId) { + this.locationId = locationId; + } + + public Integer getNoOfMembers() { + return noOfMembers; + } + + public void setNoOfMembers(Integer noOfMembers) { + this.noOfMembers = noOfMembers; + } + + public String getOrgCode() { + return orgCode; + } + + public void setOrgCode(String orgCode) { + this.orgCode = orgCode; + } + + public String getOrgName() { + return orgName; + } + + public void setOrgName(String orgName) { + this.orgName = orgName; + } + + public String getOrgType() { + return orgType; + } + + public void setOrgType(String orgType) { + this.orgType = orgType; + } + + public String getOrgTypeId() { + return orgTypeId; + } + + public void setOrgTypeId(String orgTypeId) { + this.orgTypeId = orgTypeId; + } + + public String getParentOrgId() { + return parentOrgId; + } + + public void setParentOrgId(String parentOrgId) { + this.parentOrgId = parentOrgId; + } + + public String getPreferredLanguage() { + return preferredLanguage; + } + + public void setPreferredLanguage(String preferredLanguage) { + this.preferredLanguage = preferredLanguage; + } + + public String getProvider() { + return provider; + } + + public void setProvider(String provider) { + this.provider = provider; + } + + public String getRootOrgId() { + return rootOrgId; + } + + public void setRootOrgId(String rootOrgId) { + this.rootOrgId = rootOrgId; + } + + public String getSlug() { + return slug; + } + + public void setSlug(String slug) { + this.slug = slug; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public String getTheme() { + return theme; + } + + public void setTheme(String theme) { + this.theme = theme; + } + + public String getThumbnail() { + return thumbnail; + } + + public void setThumbnail(String thumbnail) { + this.thumbnail = thumbnail; + } + + public String getUpdatedBy() { + return updatedBy; + } + + public void setUpdatedBy(String updatedBy) { + this.updatedBy = updatedBy; + } + + public String getUpdatedDate() { + return updatedDate; + } + + public void setUpdatedDate(String updatedDate) { + this.updatedDate = updatedDate; + } + + public List getLocationIds() { + return locationIds; + } + + public void setLocationIds(List locationIds) { + this.locationIds = locationIds; + } + + @JsonProperty(value = "isApproved") + public Boolean isApproved() { + return isApproved; + } + + public void setApproved(Boolean isApproved) { + this.isApproved = isApproved; + } + + @JsonProperty(value = "isDefault") + public Boolean isDefault() { + return isDefault; + } + + public void setDefault(Boolean isDefault) { + this.isDefault = isDefault; + } + + @JsonProperty(value = "isRootOrg") + public Boolean isRootOrg() { + return isRootOrg; + } + + public void setRootOrg(Boolean isRootOrg) { + this.isRootOrg = isRootOrg; + } +} diff --git a/actors/core/src/main/java/org/sunbird/learner/actors/OrganisationManagementActor.java b/actors/core/src/main/java/org/sunbird/learner/actors/OrganisationManagementActor.java index 5528c4810..d7aaa84b1 100644 --- a/actors/core/src/main/java/org/sunbird/learner/actors/OrganisationManagementActor.java +++ b/actors/core/src/main/java/org/sunbird/learner/actors/OrganisationManagementActor.java @@ -11,9 +11,12 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.sunbird.actor.core.BaseActor; import org.sunbird.actor.router.ActorConfig; +import org.sunbird.bean.Organization; import org.sunbird.cassandra.CassandraOperation; import org.sunbird.common.ElasticSearchUtil; import org.sunbird.common.exception.ProjectCommonException; @@ -62,7 +65,7 @@ asyncTasks = {} ) public class OrganisationManagementActor extends BaseActor { - + private ObjectMapper mapper = new ObjectMapper(); private final CassandraOperation cassandraOperation = ServiceFactory.getInstance(); private final EncryptionService encryptionService = org.sunbird.common.models.util.datasecurity.impl.ServiceFactory.getEncryptionServiceInstance( @@ -275,6 +278,10 @@ private void createOrg(Request actorMessage) { try { Map req = (Map) actorMessage.getRequest().get(JsonKey.ORGANISATION); + if (req.containsKey(JsonKey.LOCATION_CODE) + && !CollectionUtils.isEmpty((List) req.get(JsonKey.LOCATION_CODE))) { + validateCodeAndAddLocationIds(req); + } if (req.containsKey(JsonKey.ORG_TYPE) && !StringUtils.isBlank((String) req.get(JsonKey.ORG_TYPE))) { req.put(JsonKey.ORG_TYPE_ID, validateOrgType((String) req.get(JsonKey.ORG_TYPE))); @@ -423,7 +430,6 @@ private void createOrg(Request actorMessage) { if (req.containsKey(JsonKey.CONTACT_DETAILS)) { listOfMap = (List>) req.get(JsonKey.CONTACT_DETAILS); if (listOfMap != null && !listOfMap.isEmpty()) { - ObjectMapper mapper = new ObjectMapper(); try { req.put(JsonKey.CONTACT_DETAILS, mapper.writeValueAsString(listOfMap)); } catch (IOException e) { @@ -447,6 +453,9 @@ private void createOrg(Request actorMessage) { req.put(JsonKey.IS_ROOT_ORG, false); } + // This will remove all extra unnecessary parameter from request + Organization org = mapper.convertValue(req, Organization.class); + req = mapper.convertValue(org, Map.class); Response result = cassandraOperation.insertRecord(orgDbInfo.getKeySpace(), orgDbInfo.getTableName(), req); ProjectLogger.log("Org data saved into cassandra."); @@ -493,6 +502,13 @@ private void createOrg(Request actorMessage) { } } + private void validateCodeAndAddLocationIds(Map req) { + List locationIdList = + Util.validateLocationCode((List) req.get(JsonKey.LOCATION_CODE)); + req.put(JsonKey.LOCATION_IDS, locationIdList); + req.remove(JsonKey.LOCATION_CODE); + } + private String validateHashTagId(String hashTagId, String opType, String orgId) { Map filters = new HashMap<>(); filters.put(JsonKey.HASHTAGID, hashTagId); @@ -600,7 +616,6 @@ private void approveOrg(Request actorMessage) { if (!(list.isEmpty())) { orgDBO = list.get(0); } else { - ProjectLogger.log("Invalid Org Id"); ProjectCommonException exception = new ProjectCommonException( ResponseCode.invalidRequestData.getErrorCode(), @@ -740,6 +755,10 @@ private void updateOrgData(Request actorMessage) { try { Map req = (Map) actorMessage.getRequest().get(JsonKey.ORGANISATION); + if (req.containsKey(JsonKey.LOCATION_CODE) + && !CollectionUtils.isEmpty((List) req.get(JsonKey.LOCATION_CODE))) { + validateCodeAndAddLocationIds(req); + } if (req.containsKey(JsonKey.ORG_TYPE) && !StringUtils.isBlank((String) req.get(JsonKey.ORG_TYPE))) { req.put(JsonKey.ORG_TYPE_ID, validateOrgType((String) req.get(JsonKey.ORG_TYPE))); @@ -973,7 +992,6 @@ private void updateOrgData(Request actorMessage) { if (updateOrgDBO.containsKey(JsonKey.CONTACT_DETAILS)) { listOfMap = (List>) updateOrgDBO.get(JsonKey.CONTACT_DETAILS); if (listOfMap != null && !listOfMap.isEmpty()) { - ObjectMapper mapper = new ObjectMapper(); try { updateOrgDBO.put(JsonKey.CONTACT_DETAILS, mapper.writeValueAsString(listOfMap)); } catch (IOException e) { @@ -981,6 +999,9 @@ private void updateOrgData(Request actorMessage) { } } } + // This will remove all extra unnecessary parameter from request + Organization org = mapper.convertValue(updateOrgDBO, Organization.class); + updateOrgDBO = mapper.convertValue(org, Map.class); Response response = cassandraOperation.updateRecord( orgDbInfo.getKeySpace(), orgDbInfo.getTableName(), updateOrgDBO); @@ -1324,7 +1345,7 @@ private void getOrgDetails(Request actorMessage) { ProjectUtil.EsType.organisation.getTypeName(), orgId); - if (result == null || result.isEmpty()) { + if (MapUtils.isEmpty(result)) { throw new ProjectCommonException( ResponseCode.orgDoesNotExist.getErrorCode(), ResponseCode.orgDoesNotExist.getErrorMessage(), diff --git a/actors/core/src/main/java/org/sunbird/learner/actors/PageManagementActor.java b/actors/core/src/main/java/org/sunbird/learner/actors/PageManagementActor.java index b9d2887eb..3f3396b76 100644 --- a/actors/core/src/main/java/org/sunbird/learner/actors/PageManagementActor.java +++ b/actors/core/src/main/java/org/sunbird/learner/actors/PageManagementActor.java @@ -9,6 +9,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.sunbird.actor.core.BaseActor; import org.sunbird.actor.router.ActorConfig; @@ -17,6 +18,7 @@ import org.sunbird.common.models.response.Response; import org.sunbird.common.models.util.ActorOperations; import org.sunbird.common.models.util.JsonKey; +import org.sunbird.common.models.util.LoggerEnum; import org.sunbird.common.models.util.ProjectLogger; import org.sunbird.common.models.util.ProjectUtil; import org.sunbird.common.request.ExecutionContext; @@ -124,6 +126,7 @@ private void getSection(Request actorMessage) { } private void updatePageSection(Request actorMessage) { + ProjectLogger.log("Inside updatePageSection method", LoggerEnum.INFO); Map req = actorMessage.getRequest(); // object of telemetry event... Map targetObject = new HashMap<>(); @@ -136,7 +139,7 @@ private void updatePageSection(Request actorMessage) { sectionMap.put( JsonKey.SEARCH_QUERY, mapper.writeValueAsString(sectionMap.get(JsonKey.SEARCH_QUERY))); } catch (IOException e) { - ProjectLogger.log(e.getMessage(), e); + ProjectLogger.log("Exception occured while processing search query " + e.getMessage(), e); } } if (null != sectionMap.get(JsonKey.SECTION_DISPLAY)) { @@ -146,10 +149,11 @@ private void updatePageSection(Request actorMessage) { JsonKey.SECTION_DISPLAY, mapper.writeValueAsString(sectionMap.get(JsonKey.SECTION_DISPLAY))); } catch (IOException e) { - ProjectLogger.log(e.getMessage(), e); + ProjectLogger.log("Exception occured while processing display " + e.getMessage(), e); } } sectionMap.put(JsonKey.UPDATED_DATE, ProjectUtil.getFormattedDate()); + ProjectLogger.log("update section details", LoggerEnum.INFO); Response response = cassandraOperation.updateRecord( sectionDbInfo.getKeySpace(), sectionDbInfo.getTableName(), sectionMap); @@ -160,10 +164,12 @@ private void updatePageSection(Request actorMessage) { TelemetryUtil.telemetryProcessingCall( actorMessage.getRequest(), targetObject, correlatedObject); // update DataCacheHandler section map with updated page section data + ProjectLogger.log("Calling updateSectionDataCache method", LoggerEnum.INFO); updateSectionDataCache(response, sectionMap); } private void createPageSection(Request actorMessage) { + ProjectLogger.log("Inside createPageSection method", LoggerEnum.INFO); Map req = actorMessage.getRequest(); @SuppressWarnings("unchecked") Map sectionMap = (Map) req.get(JsonKey.SECTION); @@ -177,7 +183,7 @@ private void createPageSection(Request actorMessage) { sectionMap.put( JsonKey.SEARCH_QUERY, mapper.writeValueAsString(sectionMap.get(JsonKey.SEARCH_QUERY))); } catch (IOException e) { - ProjectLogger.log(e.getMessage(), e); + ProjectLogger.log("Exception occured while processing search Query " + e.getMessage(), e); } } if (null != sectionMap.get(JsonKey.SECTION_DISPLAY)) { @@ -185,7 +191,9 @@ private void createPageSection(Request actorMessage) { try { sectionMap.put( JsonKey.SECTION_DISPLAY, - mapper.writeValueAsString(sectionMap.get(JsonKey.SECTION_DISPLAY))); + mapper.writeValueAsString( + "Exception occured while processing search Query " + + sectionMap.get(JsonKey.SECTION_DISPLAY))); } catch (IOException e) { ProjectLogger.log(e.getMessage(), e); } @@ -203,22 +211,24 @@ private void createPageSection(Request actorMessage) { TelemetryUtil.telemetryProcessingCall( actorMessage.getRequest(), targetObject, correlatedObject); // update DataCacheHandler section map with new page section data + ProjectLogger.log("Calling updateSectionDataCache method", LoggerEnum.INFO); updateSectionDataCache(response, sectionMap); } private void updateSectionDataCache(Response response, Map sectionMap) { - new Thread() { - @Override - public void run() { - if ((JsonKey.SUCCESS).equalsIgnoreCase((String) response.get(JsonKey.RESPONSE))) { - DataCacheHandler.getSectionMap().put((String) sectionMap.get(JsonKey.ID), sectionMap); - } - } - }.start(); + new Thread( + () -> { + if ((JsonKey.SUCCESS).equalsIgnoreCase((String) response.get(JsonKey.RESPONSE))) { + DataCacheHandler.getSectionMap() + .put((String) sectionMap.get(JsonKey.ID), sectionMap); + } + }) + .start(); } @SuppressWarnings("unchecked") private void getPageData(Request actorMessage) { + ProjectLogger.log("Inside getPageData method", LoggerEnum.INFO); String sectionQuery = null; List> sectionList = new ArrayList<>(); Map filterMap = new HashMap<>(); @@ -245,15 +255,15 @@ private void getPageData(Request actorMessage) { result = (List>) response.getResult().get(JsonKey.RESPONSE); } } catch (Exception e) { - ProjectLogger.log(e.getMessage(), e); + ProjectLogger.log("Exception occured while validating org id " + e.getMessage(), e); } Map map = null; /** if orgId is not then consider default page */ - if (null == result || result.isEmpty()) { + if (CollectionUtils.isEmpty(result)) { orgId = "NA"; } - + ProjectLogger.log("Fetching data from Cache for " + orgId + ":" + pageName, LoggerEnum.INFO); Map pageMap = DataCacheHandler.getPageMap().get(orgId + ":" + pageName); if (null == pageMap) { throw new ProjectCommonException( @@ -279,8 +289,7 @@ private void getPageData(Request actorMessage) { for (Object obj : arr) { Map sectionMap = (Map) obj; Map sectionData = - new HashMap<>( - DataCacheHandler.getSectionMap().get((String) sectionMap.get(JsonKey.ID))); + new HashMap<>(DataCacheHandler.getSectionMap().get(sectionMap.get(JsonKey.ID))); getContentData(sectionData, reqFilters, headers, filterMap); sectionData.put(JsonKey.GROUP, sectionMap.get(JsonKey.GROUP)); sectionData.put(JsonKey.INDEX, sectionMap.get(JsonKey.INDEX)); @@ -335,6 +344,7 @@ private void getPageSettings() { @SuppressWarnings("unchecked") private void updatePage(Request actorMessage) { + ProjectLogger.log("Inside updatePage method", LoggerEnum.INFO); Map req = actorMessage.getRequest(); Map pageMap = (Map) req.get(JsonKey.PAGE); // object of telemetry event... @@ -354,7 +364,7 @@ private void updatePage(Request actorMessage) { pageDbInfo.getKeySpace(), pageDbInfo.getTableName(), map); if (!((List>) res.get(JsonKey.RESPONSE)).isEmpty()) { Map page = ((List>) res.get(JsonKey.RESPONSE)).get(0); - if (!(((String) page.get(JsonKey.ID)).equals((String) pageMap.get(JsonKey.ID)))) { + if (!(((String) page.get(JsonKey.ID)).equals(pageMap.get(JsonKey.ID)))) { ProjectCommonException exception = new ProjectCommonException( ResponseCode.pageAlreadyExist.getErrorCode(), @@ -371,7 +381,7 @@ private void updatePage(Request actorMessage) { try { pageMap.put(JsonKey.PORTAL_MAP, mapper.writeValueAsString(pageMap.get(JsonKey.PORTAL_MAP))); } catch (IOException e) { - ProjectLogger.log(e.getMessage(), e); + ProjectLogger.log("Exception occured while updating portal map data " + e.getMessage(), e); } } if (null != pageMap.get(JsonKey.APP_MAP)) { @@ -379,7 +389,7 @@ private void updatePage(Request actorMessage) { try { pageMap.put(JsonKey.APP_MAP, mapper.writeValueAsString(pageMap.get(JsonKey.APP_MAP))); } catch (IOException e) { - ProjectLogger.log(e.getMessage(), e); + ProjectLogger.log("Exception occured while updating app map data " + e.getMessage(), e); } } Response response = @@ -393,6 +403,8 @@ private void updatePage(Request actorMessage) { TelemetryUtil.telemetryProcessingCall( actorMessage.getRequest(), targetObject, correlatedObject); // update DataCacheHandler page map with updated page data + ProjectLogger.log( + "Calling updatePageDataCacheHandler while updating page data ", LoggerEnum.INFO); updatePageDataCacheHandler(response, pageMap); } @@ -461,19 +473,18 @@ private void createPage(Request actorMessage) { private void updatePageDataCacheHandler(Response response, Map pageMap) { // update DataCacheHandler page map with new page data - new Thread() { - @Override - public void run() { - if (JsonKey.SUCCESS.equalsIgnoreCase((String) response.get(JsonKey.RESPONSE))) { - String orgId = "NA"; - if (pageMap.containsKey(JsonKey.ORGANISATION_ID)) { - orgId = (String) pageMap.get(JsonKey.ORGANISATION_ID); - } - DataCacheHandler.getPageMap() - .put(orgId + ":" + (String) pageMap.get(JsonKey.PAGE_NAME), pageMap); - } - } - }.start(); + new Thread( + () -> { + if (JsonKey.SUCCESS.equalsIgnoreCase((String) response.get(JsonKey.RESPONSE))) { + String orgId = "NA"; + if (pageMap.containsKey(JsonKey.ORGANISATION_ID)) { + orgId = (String) pageMap.get(JsonKey.ORGANISATION_ID); + } + DataCacheHandler.getPageMap() + .put(orgId + ":" + (String) pageMap.get(JsonKey.PAGE_NAME), pageMap); + } + }) + .start(); } @SuppressWarnings("unchecked") @@ -499,7 +510,8 @@ private void getContentData( (Map) ((Map) map.get(JsonKey.REQUEST)).get(JsonKey.FILTERS); ProjectLogger.log( "default search query for ekstep for page data assemble api : " - + (String) section.get(JsonKey.SEARCH_QUERY)); + + (String) section.get(JsonKey.SEARCH_QUERY), + LoggerEnum.INFO); applyFilters(filters, reqFilters); String query = ""; @@ -512,13 +524,18 @@ private void getContentData( query = (String) section.get(JsonKey.SEARCH_QUERY); } ProjectLogger.log( - "search query after applying filter for ekstep for page data assemble api : " + query); + "search query after applying filter for ekstep for page data assemble api : " + query, + LoggerEnum.INFO); Map result = EkStepRequestUtil.searchContent(query, headers); if (null != result && !result.isEmpty()) { - section.put(JsonKey.CONTENTS, result.get(JsonKey.CONTENTS)); + section.putAll(result); + section.remove(JsonKey.PARAMS); Map tempMap = (Map) result.get(JsonKey.PARAMS); section.put(JsonKey.RES_MSG_ID, tempMap.get(JsonKey.RES_MSG_ID)); section.put(JsonKey.API_ID, tempMap.get(JsonKey.API_ID)); + } else { + ProjectLogger.log( + "Search query result from ekstep is null or empty for query " + query, LoggerEnum.INFO); } } @@ -545,8 +562,8 @@ private void applyFilters(Map filters, Map reqFi } else if (filters.get(key) instanceof Map) { filters.put(key, obj); } else { - if (!(((List) obj).contains((String) filters.get(key)))) { - ((List) obj).add((String) filters.get(key)); + if (!(((List) obj).contains(filters.get(key)))) { + ((List) obj).add(filters.get(key)); } filters.put(key, obj); } @@ -580,10 +597,10 @@ private Map getPageSetting(Map pageDO) { responseMap.put(JsonKey.ID, pageDO.get(JsonKey.ID)); if (pageDO.containsKey(JsonKey.APP_MAP) && null != pageDO.get(JsonKey.APP_MAP)) { - responseMap.put(JsonKey.APP_SECTIONS, parsePage(pageDO, (String) JsonKey.APP_MAP)); + responseMap.put(JsonKey.APP_SECTIONS, parsePage(pageDO, JsonKey.APP_MAP)); } if (pageDO.containsKey(JsonKey.PORTAL_MAP) && null != pageDO.get(JsonKey.PORTAL_MAP)) { - responseMap.put(JsonKey.PORTAL_SECTIONS, parsePage(pageDO, (String) JsonKey.PORTAL_MAP)); + responseMap.put(JsonKey.PORTAL_SECTIONS, parsePage(pageDO, JsonKey.PORTAL_MAP)); } return responseMap; } @@ -632,7 +649,7 @@ private void validateOrg(String orgId) { Response result = cassandraOperation.getRecordById(orgDbInfo.getKeySpace(), orgDbInfo.getTableName(), orgId); List> list = (List>) result.get(JsonKey.RESPONSE); - if (list == null || list.isEmpty()) { + if (CollectionUtils.isEmpty(list)) { throw new ProjectCommonException( ResponseCode.invalidOrgId.getErrorCode(), ResponseCode.invalidOrgId.getErrorMessage(), diff --git a/actors/core/src/main/java/org/sunbird/learner/actors/SchedularActor.java b/actors/core/src/main/java/org/sunbird/learner/actors/SchedularActor.java index e6fc60cc8..94f84d4f7 100644 --- a/actors/core/src/main/java/org/sunbird/learner/actors/SchedularActor.java +++ b/actors/core/src/main/java/org/sunbird/learner/actors/SchedularActor.java @@ -7,6 +7,7 @@ import org.sunbird.actor.router.ActorConfig; import org.sunbird.cassandra.CassandraOperation; import org.sunbird.common.models.util.ActorOperations; +import org.sunbird.common.models.util.BulkUploadActorOperation; import org.sunbird.common.models.util.JsonKey; import org.sunbird.common.models.util.ProjectLogger; import org.sunbird.common.models.util.ProjectUtil; @@ -66,8 +67,14 @@ private void schedule(Request request) { req.put(JsonKey.PROCESS_ID, map.get(JsonKey.ID)); ProjectLogger.log( "calling bulkUploadBackGroundJobActor for processId from schedular actor " - + map.get(JsonKey.ID)); - req.setOperation(ActorOperations.PROCESS_BULK_UPLOAD.getValue()); + + map.get(JsonKey.ID) + + " for object type:" + + map.get(JsonKey.OBJECT_TYPE)); + if (JsonKey.LOCATION.equalsIgnoreCase((String) map.get(JsonKey.OBJECT_TYPE))) { + req.setOperation(BulkUploadActorOperation.LOCATION_BULK_UPLOAD_BACKGROUND_JOB.getValue()); + } else { + req.setOperation(ActorOperations.PROCESS_BULK_UPLOAD.getValue()); + } tellToAnother(req); } } diff --git a/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/BulkUploadBackGroundJobActor.java b/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/BulkUploadBackGroundJobActor.java index 40aa64d7a..c6439c3c5 100644 --- a/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/BulkUploadBackGroundJobActor.java +++ b/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/BulkUploadBackGroundJobActor.java @@ -18,6 +18,7 @@ import org.sunbird.actor.background.BackgroundOperations; import org.sunbird.actor.core.BaseActor; import org.sunbird.actor.router.ActorConfig; +import org.sunbird.bean.Organization; import org.sunbird.cassandra.CassandraOperation; import org.sunbird.common.ElasticSearchUtil; import org.sunbird.common.exception.ProjectCommonException; @@ -77,6 +78,7 @@ public class BulkUploadBackGroundJobActor extends BaseActor { private final SSOManager ssoManager = SSOServiceFactory.getInstance(); private static final String SUNBIRD_WEB_URL = "sunbird_web_url"; private static final String SUNBIRD_APP_URL = "sunbird_app_url"; + private ObjectMapper mapper = new ObjectMapper(); @Override public void onReceive(Request request) throws Throwable { @@ -162,8 +164,8 @@ private void processBatchEnrollment(List> jsonList, String p // Insert record to BulkDb table Map map = new HashMap<>(); map.put(JsonKey.ID, processId); - map.put(JsonKey.SUCCESS_RESULT, convertMapToJsonString(successResultList)); - map.put(JsonKey.FAILURE_RESULT, convertMapToJsonString(failureResultList)); + map.put(JsonKey.SUCCESS_RESULT, ProjectUtil.convertMapToJsonString(successResultList)); + map.put(JsonKey.FAILURE_RESULT, ProjectUtil.convertMapToJsonString(failureResultList)); map.put(JsonKey.PROCESS_END_TIME, ProjectUtil.getFormattedDate()); map.put(JsonKey.STATUS, ProjectUtil.BulkProcessStatus.COMPLETED.getValue()); try { @@ -390,8 +392,8 @@ private void processOrgInfo( } } - dataMap.put(JsonKey.SUCCESS_RESULT, convertMapToJsonString(successList)); - dataMap.put(JsonKey.FAILURE_RESULT, convertMapToJsonString(failureList)); + dataMap.put(JsonKey.SUCCESS_RESULT, ProjectUtil.convertMapToJsonString(successList)); + dataMap.put(JsonKey.FAILURE_RESULT, ProjectUtil.convertMapToJsonString(failureList)); dataMap.put(JsonKey.STATUS, BulkProcessStatus.COMPLETED.getValue()); cassandraOperation.updateRecord(bulkDb.getKeySpace(), bulkDb.getTableName(), dataMap); @@ -410,10 +412,22 @@ private void processOrg( Object[] orgContactList = null; String contactDetails = null; boolean isOrgUpdated = false; + // validate location code - // object of telemetry event... - Map targetObject = new HashMap<>(); - List> correlatedObject = new ArrayList<>(); + if (concurrentHashMap.containsKey(JsonKey.LOCATION_CODE) + && StringUtils.isNotEmpty((String) concurrentHashMap.get(JsonKey.LOCATION_CODE))) { + try { + convertCommaSepStringToList(concurrentHashMap, JsonKey.LOCATION_CODE); + List locationIdList = + Util.validateLocationCode((List) concurrentHashMap.get(JsonKey.LOCATION_CODE)); + concurrentHashMap.put(JsonKey.LOCATION_IDS, locationIdList); + concurrentHashMap.remove(JsonKey.LOCATION_CODE); + } catch (Exception ex) { + concurrentHashMap.put(JsonKey.ERROR_MSG, "Invalid value for LocationCode."); + failureList.add(concurrentHashMap); + return; + } + } if (concurrentHashMap.containsKey(JsonKey.ORG_TYPE) && !ProjectUtil.isStringNullOREmpty((String) concurrentHashMap.get(JsonKey.ORG_TYPE))) { @@ -463,7 +477,6 @@ private void processOrg( contactDetails = (String) concurrentHashMap.get(JsonKey.CONTACT_DETAILS); contactDetails = contactDetails.replaceAll("'", "\""); try { - ObjectMapper mapper = new ObjectMapper(); orgContactList = mapper.readValue(contactDetails, Object[].class); } catch (IOException ex) { @@ -540,6 +553,9 @@ private void processOrg( concurrentHashMap.put(JsonKey.ID, orgResult.get(JsonKey.ID)); try { + // This will remove all extra unnecessary parameter from request + Organization org = mapper.convertValue(concurrentHashMap, Organization.class); + concurrentHashMap = mapper.convertValue(org, Map.class); cassandraOperation.upsertRecord( orgDbInfo.getKeySpace(), orgDbInfo.getTableName(), concurrentHashMap); Response orgResponse = new Response(); @@ -645,6 +661,9 @@ && isNotNull(esResult.get(JsonKey.CONTENT)) } try { + // This will remove all extra unnecessary parameter from request + Organization org = mapper.convertValue(concurrentHashMap, Organization.class); + concurrentHashMap = mapper.convertValue(org, Map.class); cassandraOperation.upsertRecord( orgDbInfo.getKeySpace(), orgDbInfo.getTableName(), concurrentHashMap); Response orgResponse = new Response(); @@ -817,6 +836,9 @@ && isNotNull(esResult.get(JsonKey.CONTENT)) concurrentHashMap.put(JsonKey.CONTACT_DETAILS, contactDetails); try { + // This will remove all extra unnecessary parameter from request + Organization org = mapper.convertValue(concurrentHashMap, Organization.class); + concurrentHashMap = mapper.convertValue(org, Map.class); cassandraOperation.upsertRecord( orgDbInfo.getKeySpace(), orgDbInfo.getTableName(), concurrentHashMap); Response orgResponse = new Response(); @@ -1113,9 +1135,11 @@ private void processUserInfo( map.put(JsonKey.ID, processId); try { map.put( - JsonKey.SUCCESS_RESULT, UserUtility.encryptData(convertMapToJsonString(successUserReq))); + JsonKey.SUCCESS_RESULT, + UserUtility.encryptData(ProjectUtil.convertMapToJsonString(successUserReq))); map.put( - JsonKey.FAILURE_RESULT, UserUtility.encryptData(convertMapToJsonString(failureUserReq))); + JsonKey.FAILURE_RESULT, + UserUtility.encryptData(ProjectUtil.convertMapToJsonString(failureUserReq))); } catch (Exception e1) { ProjectLogger.log( "Exception occurred while encrypting success and failure result in bulk upload process : ", @@ -1133,10 +1157,10 @@ private void processUserInfo( } } - private void convertCommaSepStringToList(Map userMap, String property) { - String[] userGrade = ((String) userMap.get(property)).split(","); - List list = new ArrayList<>(Arrays.asList(userGrade)); - userMap.put(property, list); + private void convertCommaSepStringToList(Map map, String property) { + String[] props = ((String) map.get(property)).split(","); + List list = new ArrayList<>(Arrays.asList(props)); + map.put(property, list); } private void updateUserOrgData(Map userMap, String updatedBy) { @@ -1198,16 +1222,6 @@ private void updateStatusForProcessing(String processId) { } } - private String convertMapToJsonString(List> mapList) { - ObjectMapper mapper = new ObjectMapper(); - try { - return mapper.writeValueAsString(mapList); - } catch (IOException e) { - ProjectLogger.log(e.getMessage(), e); - } - return null; - } - @SuppressWarnings("unchecked") private Map getBulkData(String processId) { try { diff --git a/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/BulkUploadManagementActor.java b/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/BulkUploadManagementActor.java index bd4e42a2c..6dc57e5a2 100644 --- a/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/BulkUploadManagementActor.java +++ b/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/BulkUploadManagementActor.java @@ -266,7 +266,8 @@ private void validateOrgProperty(String[] property) { JsonKey.THEME, JsonKey.CONTACT_DETAILS, JsonKey.LOC_ID, - JsonKey.HASHTAGID)); + JsonKey.HASHTAGID, + JsonKey.LOCATION_CODE)); for (String key : property) { if (!properties.contains(key)) { diff --git a/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/LocationBulkUploadActor.java b/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/LocationBulkUploadActor.java new file mode 100644 index 000000000..e096dcdac --- /dev/null +++ b/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/LocationBulkUploadActor.java @@ -0,0 +1,285 @@ +package org.sunbird.learner.actors.bulkupload; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.opencsv.CSVReader; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang.ArrayUtils; +import org.sunbird.actor.core.BaseActor; +import org.sunbird.actor.router.ActorConfig; +import org.sunbird.common.exception.ProjectCommonException; +import org.sunbird.common.models.response.Response; +import org.sunbird.common.models.util.BulkUploadActorOperation; +import org.sunbird.common.models.util.GeoLocationJsonKey; +import org.sunbird.common.models.util.JsonKey; +import org.sunbird.common.models.util.LoggerEnum; +import org.sunbird.common.models.util.ProjectLogger; +import org.sunbird.common.models.util.ProjectUtil; +import org.sunbird.common.models.util.TelemetryEnvKey; +import org.sunbird.common.request.ExecutionContext; +import org.sunbird.common.request.Request; +import org.sunbird.common.responsecode.ResponseCode; +import org.sunbird.learner.actors.bulkupload.dao.BulkUploadProcessDao; +import org.sunbird.learner.actors.bulkupload.dao.impl.BulkUploadProcessDaoImpl; +import org.sunbird.learner.actors.bulkupload.model.BulkUploadProcess; +import org.sunbird.learner.util.Util; + +/** Created by arvind on 23/4/18. */ +@ActorConfig( + tasks = {"locationBulkUpload"}, + asyncTasks = {} +) +public class LocationBulkUploadActor extends BaseActor { + + private static final String CSV_FILE_EXTENSION = ".csv"; + BulkUploadProcessDao bulkUploadDao = new BulkUploadProcessDaoImpl(); + String[] bulkLocationAllowedFields = { + GeoLocationJsonKey.CODE, + JsonKey.NAME, + GeoLocationJsonKey.PARENT_CODE, + GeoLocationJsonKey.PARENT_ID + }; + + @Override + public void onReceive(Request request) throws Throwable { + Util.initializeContext(request, TelemetryEnvKey.GEO_LOCATION); + ExecutionContext.setRequestId(request.getRequestId()); + String operation = request.getOperation(); + + switch (operation) { + case "locationBulkUpload": + upload(request); + break; + default: + onReceiveUnsupportedOperation("LocationBulkUploadActor"); + } + } + + private void upload(Request request) throws IOException { + + String processId = ProjectUtil.getUniqueIdFromTimestamp(1); + Map req = (Map) request.getRequest().get(JsonKey.DATA); + String locationType = (String) req.get(GeoLocationJsonKey.LOCATION_TYPE); + req.put(JsonKey.CREATED_BY, req.get(JsonKey.CREATED_BY)); + processLocationBulkUpload(req, processId, locationType); + } + + private void processLocationBulkUpload( + Map req, String processId, String locationType) throws IOException { + File file = new File("bulk-" + processId + CSV_FILE_EXTENSION); + List csvLines = null; + FileOutputStream fos = null; + try { + fos = new FileOutputStream(file); + fos.write((byte[]) req.get(JsonKey.FILE)); + csvLines = parseCsvFile(file); + } catch (Exception e) { + ProjectLogger.log("Exception Occurred while reading file in BulkUploadManagementActor", e); + throw e; + } finally { + try { + if (ProjectUtil.isNotNull(fos)) { + fos.close(); + } + if (ProjectUtil.isNotNull(file)) { + file.delete(); + } + } catch (IOException e) { + ProjectLogger.log( + "Exception Occurred while closing fileInputStream in BulkUploadManagementActor", e); + } + } + + validateBulkUploadSize(csvLines); + validateBulkUploadFields(csvLines.get(0), bulkLocationAllowedFields); + // save csv file to db + BulkUploadProcess bulkUploadProcess = + uploadCsvToDB( + csvLines, + processId, + JsonKey.LOCATION, + (String) req.get(JsonKey.CREATED_BY), + locationType); + Response res = bulkUploadDao.create(bulkUploadProcess); + + sender().tell(res, self()); + if (((String) res.get(JsonKey.RESPONSE)).equalsIgnoreCase(JsonKey.SUCCESS)) { + Request request = new Request(); + request.put(JsonKey.PROCESS_ID, processId); + request.setOperation(BulkUploadActorOperation.LOCATION_BULK_UPLOAD_BACKGROUND_JOB.getValue()); + ProjectLogger.log( + "LocationBulkUploadActor : calling action" + + BulkUploadActorOperation.LOCATION_BULK_UPLOAD_BACKGROUND_JOB.getValue()); + tellToAnother(request); + } + } + + private BulkUploadProcess uploadCsvToDB( + List csvLines, + String processId, + String objectType, + String requestedBy, + String locationType) { + + ProjectLogger.log("LocationBulkUploadActor : uploadCsvToDB method started", LoggerEnum.INFO); + List> dataMapList = new ArrayList<>(); + if (csvLines.size() > 1) { + try { + String[] columnArr = csvLines.get(0); + columnArr = trimColumnAttriutes(columnArr); + Map dataMap = null; + for (int i = 1; i < csvLines.size(); i++) { + dataMap = new HashMap<>(); + String[] valueArr = csvLines.get(i); + for (int j = 0; j < valueArr.length; j++) { + String value = (valueArr[j].trim().length() == 0 ? null : valueArr[j].trim()); + dataMap.put(columnArr[j], value); + } + dataMap.put(GeoLocationJsonKey.LOCATION_TYPE, locationType); + dataMapList.add(dataMap); + } + } catch (Exception e) { + ProjectLogger.log( + "LocationBulkUploadActor : error while uploading csv to db" + e.getMessage(), e); + throw new ProjectCommonException( + ResponseCode.csvError.getErrorCode(), + ResponseCode.csvError.getErrorMessage(), + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + } else { + ProjectLogger.log("CSV size error ", LoggerEnum.INFO); + throw new ProjectCommonException( + ResponseCode.csvError.getErrorCode(), + ResponseCode.csvError.getErrorMessage(), + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + // convert userMapList to json string + Map map = new HashMap<>(); + + ObjectMapper mapper = new ObjectMapper(); + try { + map.put(JsonKey.DATA, mapper.writeValueAsString(dataMapList)); + } catch (Exception e) { + ProjectLogger.log( + "LocationBulkUploadActor : exception while converting data map list to string " + + e.getMessage(), + e); + throw new ProjectCommonException( + ResponseCode.unableToParseData.getErrorCode(), + ResponseCode.unableToParseData.getErrorMessage(), + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + + BulkUploadProcess bulkUploadProcess = new BulkUploadProcess(); + bulkUploadProcess.setId(processId); + try { + bulkUploadProcess.setData(mapper.writeValueAsString(dataMapList)); + } catch (Exception e) { + ProjectLogger.log( + "LocationBulkUploadActor : exception while setting data to bulkUploadProcess Pojo " + + e.getMessage(), + e); + throw new ProjectCommonException( + ResponseCode.unableToParseData.getErrorCode(), + ResponseCode.unableToParseData.getErrorMessage(), + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + + bulkUploadProcess.setId(processId); + bulkUploadProcess.setObjectType(objectType); + bulkUploadProcess.setUploadedBy(requestedBy); + bulkUploadProcess.setUploadedDate(ProjectUtil.getFormattedDate()); + bulkUploadProcess.setProcessStartTime(ProjectUtil.getFormattedDate()); + bulkUploadProcess.setStatus(ProjectUtil.BulkProcessStatus.NEW.getValue()); + return bulkUploadProcess; + } + + private void validateBulkUploadSize(List csvLines) { + + if (null != csvLines) { + if (csvLines.size() < 2) { + throw new ProjectCommonException( + ResponseCode.emptyFile.getErrorCode(), + ResponseCode.emptyFile.getErrorMessage(), + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + } else { + throw new ProjectCommonException( + ResponseCode.emptyFile.getErrorCode(), + ResponseCode.emptyFile.getErrorMessage(), + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + } + + private void validateBulkUploadFields( + String[] csvHeaderLine, String[] bulkLocationAllowedFields) { + + if (ArrayUtils.isEmpty(csvHeaderLine) + || ArrayUtils.isEmpty(bulkLocationAllowedFields) + || !ArrayUtils.isSameLength(csvHeaderLine, bulkLocationAllowedFields)) { + throw new ProjectCommonException( + ResponseCode.invalidColumns.getErrorCode(), + ResponseCode.invalidColumns.getErrorMessage(), + ResponseCode.CLIENT_ERROR.getResponseCode(), + String.join(",", bulkLocationAllowedFields)); + } + + Arrays.stream(bulkLocationAllowedFields) + .forEach( + x -> { + if (!(ArrayUtils.contains(csvHeaderLine, x))) { + throw new ProjectCommonException( + ResponseCode.invalidColumns.getErrorCode(), + ResponseCode.invalidColumns.getErrorMessage(), + ResponseCode.CLIENT_ERROR.getResponseCode(), + String.join(",", bulkLocationAllowedFields)); + } + }); + } + + private List parseCsvFile(File file) { + CSVReader csvReader = null; + List lines = new ArrayList<>(); + try { + // Reading the csv file + csvReader = new CSVReader(new FileReader(file), ',', '"', 0); + String[] csvLine; + while ((csvLine = csvReader.readNext()) != null) { + if (ProjectUtil.isNotEmptyStringArray(csvLine)) { + continue; + } + List list = new ArrayList<>(); + for (String csvLineColumn : csvLine) { + list.add(csvLineColumn); + } + lines.add(list.toArray(list.toArray(new String[csvLine.length]))); + } + } catch (Exception e) { + ProjectLogger.log("Exception occurred while processing csv file : ", e); + } finally { + try { + // closing the reader + csvReader.close(); + file.delete(); + } catch (Exception e) { + ProjectLogger.log("Exception occurred while closing csv reader : ", e); + } + } + return lines; + } + + private String[] trimColumnAttriutes(String[] columnArr) { + + for (int i = 0; i < columnArr.length; i++) { + columnArr[i] = columnArr[i].trim(); + } + return columnArr; + } +} diff --git a/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/LocationBulkUploadBackGroundJobActor.java b/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/LocationBulkUploadBackGroundJobActor.java new file mode 100644 index 000000000..0488a4beb --- /dev/null +++ b/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/LocationBulkUploadBackGroundJobActor.java @@ -0,0 +1,229 @@ +package org.sunbird.learner.actors.bulkupload; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.text.MessageFormat; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import org.apache.commons.collections.CollectionUtils; +import org.sunbird.actor.core.BaseActor; +import org.sunbird.actor.core.service.InterServiceCommunication; +import org.sunbird.actor.core.service.InterServiceCommunicationFactory; +import org.sunbird.actor.router.ActorConfig; +import org.sunbird.common.exception.ProjectCommonException; +import org.sunbird.common.models.response.Response; +import org.sunbird.common.models.util.GeoLocationJsonKey; +import org.sunbird.common.models.util.JsonKey; +import org.sunbird.common.models.util.LocationActorOperation; +import org.sunbird.common.models.util.LoggerEnum; +import org.sunbird.common.models.util.ProjectLogger; +import org.sunbird.common.models.util.ProjectUtil; +import org.sunbird.common.models.util.ProjectUtil.BulkProcessStatus; +import org.sunbird.common.models.util.TelemetryEnvKey; +import org.sunbird.common.request.ExecutionContext; +import org.sunbird.common.request.Request; +import org.sunbird.common.responsecode.ResponseCode; +import org.sunbird.learner.actors.bulkupload.dao.BulkUploadProcessDao; +import org.sunbird.learner.actors.bulkupload.dao.impl.BulkUploadProcessDaoImpl; +import org.sunbird.learner.actors.bulkupload.model.BulkUploadProcess; +import org.sunbird.learner.util.Util; + +/** Created by arvind on 24/4/18. */ +@ActorConfig( + tasks = {}, + asyncTasks = {"locationBulkUploadBackground"} +) +public class LocationBulkUploadBackGroundJobActor extends BaseActor { + + BulkUploadProcessDao bulkUploadDao = new BulkUploadProcessDaoImpl(); + ObjectMapper mapper = new ObjectMapper(); + InterServiceCommunication interServiceCommunication = + InterServiceCommunicationFactory.getInstance().getCommunicationPath("actorCommunication"); + + @Override + public void onReceive(Request request) throws Throwable { + + String operation = request.getOperation(); + Util.initializeContext(request, TelemetryEnvKey.GEO_LOCATION); + ExecutionContext.setRequestId(request.getRequestId()); + + switch (operation) { + case "locationBulkUploadBackground": + bulkLocationUpload(request); + break; + default: + onReceiveUnsupportedOperation("LocationBulkUploadBackGroundJobActor"); + } + } + + private void bulkLocationUpload(Request request) throws IOException { + + String processId = (String) request.get(JsonKey.PROCESS_ID); + BulkUploadProcess bulkUploadProcess = bulkUploadDao.read(processId); + if (null == bulkUploadProcess) { + ProjectLogger.log("Process Id does not exist : " + processId, LoggerEnum.ERROR); + return; + } + Integer status = bulkUploadProcess.getStatus(); + if (!(status == (ProjectUtil.BulkProcessStatus.COMPLETED.getValue()) + || status == (ProjectUtil.BulkProcessStatus.INTERRUPT.getValue()))) { + processLocationBulkUpoad(bulkUploadProcess); + } + } + + private void processLocationBulkUpoad(BulkUploadProcess bulkUploadProcess) throws IOException { + + TypeReference>> mapType = + new TypeReference>>() {}; + List> jsonList = new LinkedList<>(); + List> successList = new LinkedList<>(); + List> failureList = new LinkedList<>(); + try { + jsonList = mapper.readValue(bulkUploadProcess.getData(), mapType); + } catch (Exception e) { + ProjectLogger.log( + "LocationBulkUploadBackGroundJobActor : Exception occurred while converting json String to List:", + e); + throw e; + } + + for (Map row : jsonList) { + processLocation(row, successList, failureList); + } + + ProjectLogger.log( + "LocationBulkUploadBackGroundJobActor : processLocationBulkUpoad process finished", + LoggerEnum.INFO); + bulkUploadProcess.setSuccessResult(ProjectUtil.convertMapToJsonString(successList)); + bulkUploadProcess.setFailureResult(ProjectUtil.convertMapToJsonString(failureList)); + bulkUploadProcess.setStatus(BulkProcessStatus.COMPLETED.getValue()); + bulkUploadDao.update(bulkUploadProcess); + } + + private void processLocation( + Map row, + List> successList, + List> failureList) { + + ProjectLogger.log( + "LocationBulkUploadBackGroundJobActor : processLocation method called", LoggerEnum.INFO); + + if (checkMandatoryFields(row, GeoLocationJsonKey.CODE)) { + Request request = new Request(); + Map filters = new HashMap<>(); + filters.put(GeoLocationJsonKey.CODE, row.get(GeoLocationJsonKey.CODE)); + filters.put(GeoLocationJsonKey.LOCATION_TYPE, row.get(GeoLocationJsonKey.LOCATION_TYPE)); + request.getRequest().put(JsonKey.FILTERS, filters); + + Object obj = + interServiceCommunication.getResponse( + request, LocationActorOperation.SEARCH_LOCATION.getValue()); + if (null == obj) { + ProjectLogger.log("Null receive from interservice communication", LoggerEnum.ERROR); + failureList.add(row); + } else if (obj instanceof ProjectCommonException) { + row.put(JsonKey.ERROR_MSG, ((ProjectCommonException) obj).getMessage()); + failureList.add(row); + } else if (obj instanceof Response) { + Response response = (Response) obj; + List> responseList = + (List>) response.getResult().get(JsonKey.RESPONSE); + if (CollectionUtils.isEmpty(responseList)) { + callCreateLocation(row, successList, failureList); + } else { + callUpdateLocation(row, successList, failureList, responseList.get(0)); + } + } + } else { + row.put( + JsonKey.ERROR_MSG, + MessageFormat.format( + ResponseCode.mandatoryParamsMissing.getErrorMessage(), GeoLocationJsonKey.CODE)); + failureList.add(row); + } + } + + private boolean checkMandatoryFields(Map row, String... fields) { + + boolean flag = true; + for (String field : fields) { + if (!(row.containsKey(field))) { + flag = false; + break; + } + } + return flag; + } + + private void callUpdateLocation( + Map row, + List> successList, + List> failureList, + Map response) { + + String id = (String) response.get(JsonKey.ID); + row.put(JsonKey.ID, id); + + Request request = new Request(); + request.getRequest().putAll(row); + ProjectLogger.log( + "callUpdateLocation - " + + (request instanceof Request) + + "Operation -" + + LocationActorOperation.UPDATE_LOCATION.getValue(), + LoggerEnum.INFO); + + Object obj = + interServiceCommunication.getResponse( + request, LocationActorOperation.UPDATE_LOCATION.getValue()); + + if (null == obj) { + ProjectLogger.log("Null receive from interservice communication", LoggerEnum.ERROR); + failureList.add(row); + } else if (obj instanceof ProjectCommonException) { + ProjectLogger.log( + "callUpdateLocation - got exception from UpdateLocationService " + + ((ProjectCommonException) obj).getMessage(), + LoggerEnum.INFO); + row.put(JsonKey.ERROR_MSG, ((ProjectCommonException) obj).getMessage()); + failureList.add(row); + } else if (obj instanceof Response) { + successList.add(row); + } + } + + private void callCreateLocation( + Map row, + List> successList, + List> failureList) { + + Request request = new Request(); + request.getRequest().putAll(row); + ProjectLogger.log( + "callCreateLocation - " + + (request instanceof Request) + + "Operation -" + + LocationActorOperation.CREATE_LOCATION.getValue(), + LoggerEnum.INFO); + Object obj = + interServiceCommunication.getResponse( + request, LocationActorOperation.CREATE_LOCATION.getValue()); + + if (null == obj) { + ProjectLogger.log("Null receive from interservice communication", LoggerEnum.ERROR); + failureList.add(row); + } else if (obj instanceof ProjectCommonException) { + ProjectLogger.log( + "callCreateLocation - got exception from CreateLocationService " + + ((ProjectCommonException) obj).getMessage(), + LoggerEnum.INFO); + row.put(JsonKey.ERROR_MSG, ((ProjectCommonException) obj).getMessage()); + failureList.add(row); + } else if (obj instanceof Response) { + successList.add(row); + } + } +} diff --git a/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/dao/BulkUploadProcessDao.java b/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/dao/BulkUploadProcessDao.java new file mode 100644 index 000000000..22ddd509d --- /dev/null +++ b/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/dao/BulkUploadProcessDao.java @@ -0,0 +1,26 @@ +package org.sunbird.learner.actors.bulkupload.dao; + +import org.sunbird.common.models.response.Response; +import org.sunbird.learner.actors.bulkupload.model.BulkUploadProcess; + +/** Created by arvind on 24/4/18. */ +public interface BulkUploadProcessDao { + + /** + * @param bulkUploadProcess + * @return response Response + */ + Response create(BulkUploadProcess bulkUploadProcess); + + /** + * @param bulkUploadProcess + * @return response Response + */ + Response update(BulkUploadProcess bulkUploadProcess); + + /** + * @param id + * @return response Response + */ + BulkUploadProcess read(String id); +} diff --git a/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/dao/impl/BulkUploadProcessDaoImpl.java b/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/dao/impl/BulkUploadProcessDaoImpl.java new file mode 100644 index 000000000..cb7c91f58 --- /dev/null +++ b/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/dao/impl/BulkUploadProcessDaoImpl.java @@ -0,0 +1,54 @@ +package org.sunbird.learner.actors.bulkupload.dao.impl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import org.apache.commons.collections.CollectionUtils; +import org.sunbird.cassandra.CassandraOperation; +import org.sunbird.common.models.response.Response; +import org.sunbird.common.models.util.JsonKey; +import org.sunbird.common.models.util.ProjectLogger; +import org.sunbird.helper.ServiceFactory; +import org.sunbird.learner.actors.bulkupload.dao.BulkUploadProcessDao; +import org.sunbird.learner.actors.bulkupload.model.BulkUploadProcess; + +/** Created by arvind on 24/4/18. */ +public class BulkUploadProcessDaoImpl implements BulkUploadProcessDao { + + private CassandraOperation cassandraOperation = ServiceFactory.getInstance(); + private ObjectMapper mapper = new ObjectMapper(); + private static final String KEYSPACE_NAME = "sunbird"; + private static final String TABLE_NAME = "bulk_upload_process"; + + @Override + public Response create(BulkUploadProcess bulkUploadProcess) { + Map map = mapper.convertValue(bulkUploadProcess, Map.class); + Response response = cassandraOperation.insertRecord(KEYSPACE_NAME, TABLE_NAME, map); + // need to send ID along with success msg + response.put(JsonKey.ID, map.get(JsonKey.ID)); + return response; + } + + @Override + public Response update(BulkUploadProcess bulkUploadProcess) { + Map map = mapper.convertValue(bulkUploadProcess, Map.class); + return cassandraOperation.updateRecord(KEYSPACE_NAME, TABLE_NAME, map); + } + + @Override + public BulkUploadProcess read(String id) { + Response response = cassandraOperation.getRecordById(KEYSPACE_NAME, TABLE_NAME, id); + List> list = (List>) response.get(JsonKey.RESPONSE); + if (CollectionUtils.isEmpty(list)) { + return null; + } + try { + String jsonString = mapper.writeValueAsString((Map) list.get(0)); + return mapper.readValue(jsonString, BulkUploadProcess.class); + } catch (IOException e) { + ProjectLogger.log(e.getMessage(), e); + } + return null; + } +} diff --git a/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/model/BulkUploadProcess.java b/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/model/BulkUploadProcess.java new file mode 100644 index 000000000..70fa820b1 --- /dev/null +++ b/actors/core/src/main/java/org/sunbird/learner/actors/bulkupload/model/BulkUploadProcess.java @@ -0,0 +1,123 @@ +package org.sunbird.learner.actors.bulkupload.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import java.io.Serializable; + +/** Created by arvind on 24/4/18. */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(Include.NON_NULL) +public class BulkUploadProcess implements Serializable { + + private static final long serialVersionUID = 1L; + + private String id; + private String data; + private String failureResult; + private String objectType; + private String organisationId; + private String processEndTime; + private String processStartTime; + private Integer retryCount; + private Integer status; + private String successResult; + private String uploadedBy; + private String uploadedDate; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getData() { + return data; + } + + public void setData(String data) { + this.data = data; + } + + public String getFailureResult() { + return failureResult; + } + + public void setFailureResult(String failureResult) { + this.failureResult = failureResult; + } + + public String getObjectType() { + return objectType; + } + + public void setObjectType(String objectType) { + this.objectType = objectType; + } + + public String getOrganisationId() { + return organisationId; + } + + public void setOrganisationId(String organisationId) { + this.organisationId = organisationId; + } + + public Integer getRetryCount() { + return retryCount; + } + + public void setRetryCount(Integer retryCount) { + this.retryCount = retryCount; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public String getSuccessResult() { + return successResult; + } + + public void setSuccessResult(String successResult) { + this.successResult = successResult; + } + + public String getUploadedBy() { + return uploadedBy; + } + + public void setUploadedBy(String uploadedBy) { + this.uploadedBy = uploadedBy; + } + + public String getUploadedDate() { + return uploadedDate; + } + + public void setUploadedDate(String uploadedDate) { + this.uploadedDate = uploadedDate; + } + + public String getProcessEndTime() { + return processEndTime; + } + + public void setProcessEndTime(String processEndTime) { + this.processEndTime = processEndTime; + } + + public String getProcessStartTime() { + return processStartTime; + } + + public void setProcessStartTime(String processStartTime) { + this.processStartTime = processStartTime; + } +} diff --git a/actors/core/src/main/java/org/sunbird/learner/actors/syncjobmanager/EsSyncActor.java b/actors/core/src/main/java/org/sunbird/learner/actors/syncjobmanager/EsSyncActor.java index 226a16ae3..786fd0be4 100644 --- a/actors/core/src/main/java/org/sunbird/learner/actors/syncjobmanager/EsSyncActor.java +++ b/actors/core/src/main/java/org/sunbird/learner/actors/syncjobmanager/EsSyncActor.java @@ -18,6 +18,7 @@ import org.sunbird.common.models.util.ActorOperations; import org.sunbird.common.models.util.BadgingJsonKey; import org.sunbird.common.models.util.JsonKey; +import org.sunbird.common.models.util.LoggerEnum; import org.sunbird.common.models.util.ProjectLogger; import org.sunbird.common.models.util.ProjectUtil; import org.sunbird.common.models.util.PropertiesCache; @@ -59,7 +60,6 @@ public class EsSyncActor extends BaseActor { @Override public void onReceive(Request request) throws Throwable { String requestedOperation = request.getOperation(); - ProjectLogger.log("Operation name is ==" + requestedOperation); if (requestedOperation.equalsIgnoreCase(ActorOperations.SYNC.getValue())) { // return SUCCESS to controller and run the sync process in background Response response = new Response(); @@ -72,7 +72,7 @@ public void onReceive(Request request) throws Throwable { } private void syncData(Request message) { - ProjectLogger.log("DB data sync operation to elastic search started "); + ProjectLogger.log("DB data sync operation to elastic search started ", LoggerEnum.INFO); long startTime = System.currentTimeMillis(); Map req = message.getRequest(); Map responseMap = new HashMap<>(); @@ -97,7 +97,8 @@ private void syncData(Request message) { + objectType + " for these ids " + Arrays.toString(objectIds.toArray()) - + " started"); + + " started", + LoggerEnum.INFO); Response response = cassandraOperation.getRecordsByProperty( dbInfo.getKeySpace(), dbInfo.getTableName(), JsonKey.ID, objectIds); @@ -107,20 +108,22 @@ private void syncData(Request message) { + objectType + " for these ids " + Arrays.toString(objectIds.toArray()) - + " done"); + + " done", + LoggerEnum.INFO); } if (null != reponseList && !reponseList.isEmpty()) { for (Map map : reponseList) { responseMap.put((String) map.get(JsonKey.ID), map); } } else { - ProjectLogger.log("fetching all data for " + objectType + " started"); + ProjectLogger.log("fetching all data for " + objectType + " started", LoggerEnum.INFO); Response response = cassandraOperation.getAllRecords(dbInfo.getKeySpace(), dbInfo.getTableName()); reponseList = (List>) response.get(JsonKey.RESPONSE); - ProjectLogger.log("fetching all data for " + objectType + " done"); + ProjectLogger.log("fetching all data for " + objectType + " done", LoggerEnum.INFO); ProjectLogger.log( - "total db data to sync for " + objectType + " to Elastic search " + reponseList.size()); + "total db data to sync for " + objectType + " to Elastic search " + reponseList.size(), + LoggerEnum.INFO); if (null != reponseList) { for (Map map : reponseList) { responseMap.put((String) map.get(JsonKey.ID), map); @@ -132,7 +135,11 @@ private void syncData(Request message) { if (objectType.equals(JsonKey.USER)) { Entry entry = itr.next(); Map userMap = (Map) entry.getValue(); - if (!((boolean) userMap.get(JsonKey.IS_DELETED))) { + Boolean isDeleted = false; + if (null != userMap.get(JsonKey.IS_DELETED)) { + isDeleted = (Boolean) userMap.get(JsonKey.IS_DELETED); + } + if (!isDeleted) { result.add(getUserDetails(entry)); } } else if (objectType.equals(JsonKey.ORGANISATION)) { @@ -151,7 +158,8 @@ private void syncData(Request message) { + objectType + " to Elastic search " + elapsedTime - + " ms."); + + " ms.", + LoggerEnum.INFO); } private String getType(String objectType) { @@ -169,7 +177,7 @@ private String getType(String objectType) { } private Map getOrgDetails(Entry entry) { - ProjectLogger.log("fetching org data started"); + ProjectLogger.log("fetching org data started", LoggerEnum.INFO); Map orgMap = (Map) entry.getValue(); orgMap.remove(JsonKey.ORG_TYPE); if (orgMap.containsKey(JsonKey.ADDRESS_ID) @@ -179,14 +187,14 @@ private Map getOrgDetails(Entry entry) { getDetailsById( Util.dbInfoMap.get(JsonKey.ADDRESS_DB), (String) orgMap.get(JsonKey.ADDRESS_ID))); } - ProjectLogger.log("fetching org data completed"); + ProjectLogger.log("fetching org data completed", LoggerEnum.INFO); return orgMap; } @SuppressWarnings("unchecked") private Map getUserDetails(Entry entry) { String userId = entry.getKey(); - ProjectLogger.log("fetching user data started"); + ProjectLogger.log("fetching user data started", LoggerEnum.INFO); Map userMap = (Map) entry.getValue(); Util.removeAttributes(userMap, Arrays.asList(JsonKey.PASSWORD, JsonKey.UPDATED_BY)); if (StringUtils.isBlank((String) userMap.get(JsonKey.COUNTRY_CODE))) { @@ -194,19 +202,19 @@ private Map getUserDetails(Entry entry) { JsonKey.COUNTRY_CODE, PropertiesCache.getInstance().getProperty("sunbird_default_country_code")); } - ProjectLogger.log("fetching user address data started"); + ProjectLogger.log("fetching user address data started", LoggerEnum.INFO); String encryption = PropertiesCache.getInstance().getProperty(JsonKey.SUNBIRD_ENCRYPTION); String uid = userId; uid = encryptUserData(encryption, uid); userMap.put( JsonKey.ADDRESS, getDetails(Util.dbInfoMap.get(JsonKey.ADDRESS_DB), uid, JsonKey.USER_ID)); - ProjectLogger.log("fetching user education data started"); + ProjectLogger.log("fetching user education data started", LoggerEnum.INFO); fetchEducationDetails(userId, userMap); - ProjectLogger.log("fetching user job profile data started"); + ProjectLogger.log("fetching user job profile data started", LoggerEnum.INFO); fetchJobDetails(userId, userMap); - ProjectLogger.log("fetching user org data started"); + ProjectLogger.log("fetching user org data started", LoggerEnum.INFO); fetchUserOrgDetails(userId, userMap); - ProjectLogger.log("fetching user Badge data started"); + ProjectLogger.log("fetching user Badge data started", LoggerEnum.INFO); fetchUserBadgeDetails(userId, userMap); // save masked email and phone number @@ -215,7 +223,7 @@ private Map getUserDetails(Entry entry) { fetchUserSkills(userId, userMap); // compute profile completeness and error field. checkProfileCompleteness(userId, userMap); - ProjectLogger.log("fetching user data completed"); + ProjectLogger.log("fetching user data completed", LoggerEnum.INFO); return userMap; } diff --git a/actors/core/src/main/java/org/sunbird/learner/util/EkStepRequestUtil.java b/actors/core/src/main/java/org/sunbird/learner/util/EkStepRequestUtil.java index f19704c89..e8fa293cf 100644 --- a/actors/core/src/main/java/org/sunbird/learner/util/EkStepRequestUtil.java +++ b/actors/core/src/main/java/org/sunbird/learner/util/EkStepRequestUtil.java @@ -4,9 +4,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import org.apache.commons.lang3.StringUtils; -import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.sunbird.common.models.util.HttpUtil; @@ -35,7 +35,7 @@ public static Map searchContent(String params, Map resMap = new HashMap<>(); String response = ""; - JSONObject data; + JSONObject jObject; try { String baseSearchUrl = System.getenv(JsonKey.EKSTEP_BASE_URL); @@ -45,7 +45,7 @@ public static Map searchContent(String params, Map searchContent(String params, Map data = mapper.readValue(resultStr, Map.class); ProjectLogger.log( "Total number of content fetched from Ekstep while assembling page data : " + data.get("count"), LoggerEnum.INFO.name()); - JSONArray contentArray = data.getJSONArray(JsonKey.CONTENT); - result = mapper.readValue(contentArray.toString(), Object[].class); + Object contentList = data.get(JsonKey.CONTENT); Map param = new HashMap<>(); param.put(JsonKey.RES_MSG_ID, resmsgId); param.put(JsonKey.API_ID, apiId); resMap.put(JsonKey.PARAMS, param); - resMap.put(JsonKey.CONTENTS, result); + resMap.put(JsonKey.CONTENTS, contentList); + Iterator> itr = data.entrySet().iterator(); + while (itr.hasNext()) { + Map.Entry entry = itr.next(); + if (!JsonKey.CONTENT.equals(entry.getKey())) { + resMap.put(entry.getKey(), entry.getValue()); + } + } } catch (IOException | JSONException e) { ProjectLogger.log("Error found during contnet search parse==" + e.getMessage(), e); } diff --git a/actors/core/src/main/java/org/sunbird/learner/util/Util.java b/actors/core/src/main/java/org/sunbird/learner/util/Util.java index b7f610927..e76394b31 100644 --- a/actors/core/src/main/java/org/sunbird/learner/util/Util.java +++ b/actors/core/src/main/java/org/sunbird/learner/util/Util.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -14,6 +15,7 @@ import java.util.Map.Entry; import java.util.Properties; import java.util.Set; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.json.JSONArray; import org.json.JSONException; @@ -24,6 +26,7 @@ import org.sunbird.common.models.response.Response; import org.sunbird.common.models.util.ActorOperations; import org.sunbird.common.models.util.BadgingJsonKey; +import org.sunbird.common.models.util.GeoLocationJsonKey; import org.sunbird.common.models.util.HttpUtil; import org.sunbird.common.models.util.JsonKey; import org.sunbird.common.models.util.LoggerEnum; @@ -889,4 +892,63 @@ private static Map getOrgDetails(String identifier) { } return Collections.emptyMap(); } + + /** + * This method will validate the list of location code whether its valid or not. If valid will + * return the locationId List. + * + * @param codeList + */ + public static List validateLocationCode(List codeList) { + Map filters = new HashMap<>(); + filters.put(GeoLocationJsonKey.CODE, codeList); + Map locMap = new HashMap<>(); + locMap.put(JsonKey.FILTERS, filters); + List> locationList = + getESSearchResult( + locMap, + ProjectUtil.EsIndex.sunbird.getIndexName(), + ProjectUtil.EsType.location.getTypeName()); + List locationIdList = new ArrayList<>(); + if (!CollectionUtils.isEmpty(locationList)) { + List responseLocCodeList = new ArrayList<>(); + for (Map map : locationList) { + responseLocCodeList.add(((String) map.get(JsonKey.CODE)).toLowerCase()); + locationIdList.add((String) map.get(JsonKey.ID)); + } + List invalidValueList = new ArrayList<>(); + for (Object code : codeList) { + String loc = (String) code; + if (!responseLocCodeList.contains(loc.toLowerCase())) { + invalidValueList.add(loc); + } + } + if (CollectionUtils.isNotEmpty(invalidValueList)) { + throw new ProjectCommonException( + ResponseCode.invalidParameterValue.getErrorCode(), + ProjectUtil.formatMessage( + ResponseCode.invalidParameterValue.getErrorMessage(), + invalidValueList, + JsonKey.LOCATION_CODE), + ResponseCode.CLIENT_ERROR.getResponseCode()); + } else { + return locationIdList; + } + } else { + throw new ProjectCommonException( + ResponseCode.invalidParameterValue.getErrorCode(), + ProjectUtil.formatMessage( + ResponseCode.invalidParameterValue.getErrorMessage(), + codeList, + JsonKey.LOCATION_CODE), + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + } + + public static List> getESSearchResult( + Map searchQueryMap, String esIndex, String esType) { + SearchDTO searchDto = Util.createSearchDto(searchQueryMap); + Map result = ElasticSearchUtil.complexSearch(searchDto, esIndex, esType); + return (List>) result.get(JsonKey.CONTENT); + } } diff --git a/actors/core/src/main/java/org/sunbird/telemetry/TelemetryActor.java b/actors/core/src/main/java/org/sunbird/telemetry/TelemetryActor.java deleted file mode 100644 index 79cd2373b..000000000 --- a/actors/core/src/main/java/org/sunbird/telemetry/TelemetryActor.java +++ /dev/null @@ -1,59 +0,0 @@ -/** */ -package org.sunbird.telemetry; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import org.sunbird.actor.core.BaseActor; -import org.sunbird.actor.router.ActorConfig; -import org.sunbird.common.models.response.Response; -import org.sunbird.common.models.util.JsonKey; -import org.sunbird.common.models.util.LoggerEnum; -import org.sunbird.common.models.util.ProjectLogger; -import org.sunbird.common.request.Request; -import org.sunbird.util.lmaxdisruptor.LMAXWriter; - -/** @author Manzarul */ -@ActorConfig( - tasks = {"saveTelemetry"}, - asyncTasks = {} -) -public class TelemetryActor extends BaseActor { - - @Override - public void onReceive(Request request) throws Throwable { - ProjectLogger.log("TelemetryActor onReceive called", LoggerEnum.INFO.name()); - String operation = request.getOperation(); - switch (operation) { - case "saveTelemetry": - saveTelemetry(request); - break; - default: - onReceiveUnsupportedOperation("TelemetryActor"); - } - } - - /** - * This method will call the badger server to create badge assertion. - * - * @param request Request - */ - private void saveTelemetry(Request request) throws IOException { - ProjectLogger.log("Saving telemetry data.", LoggerEnum.DEBUG.name()); - request.put(JsonKey.HEADER, createHeader(request)); - LMAXWriter.getInstance().submitMessage(request); - sender().tell(new Response(), self()); - } - - private Map createHeader(Request request) { - Map map = new HashMap<>(); - String authKey = System.getenv("ekstep_authorization"); - map.put("authorization", JsonKey.BEARER + authKey); - if (request.getRequest().containsKey(JsonKey.FILE)) { - map.put(JsonKey.CONTENT_ENCODING, "gzip"); - } else { - map.put("Content-Type", "application/json"); - } - return map; - } -} diff --git a/actors/location/pom.xml b/actors/location/pom.xml new file mode 100644 index 000000000..d1ece81d3 --- /dev/null +++ b/actors/location/pom.xml @@ -0,0 +1,106 @@ + + + 4.0.0 + + org.sunbird + mw-actors + 1.0-SNAPSHOT + ../pom.xml + + location + Location + + UTF-8 + + + + org.sunbird + actor-core + 1.0-SNAPSHOT + + + com.typesafe.akka + akka-testkit_2.11 + 2.5.3 + test + + + junit + junit + 4.12 + test + + + org.powermock + powermock-module-junit4 + 1.6.5 + test + + + junit + junit + + + + + org.powermock + powermock-api-mockito + 1.6.5 + test + + + + ${basedir}/src/main/java + ${basedir}/src/test/java + + + org.jacoco + jacoco-maven-plugin + 0.7.5.201505241946 + + ${basedir}/target/coverage-reports/jacoco-unit.exec + ${basedir}/target/coverage-reports/jacoco-unit.exec + + + + jacoco-initialize + + prepare-agent + + + + jacoco-site + package + + report + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 2.3.2 + + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.20 + + + + **/*Spec.java + **/*Test.java + + + + + + diff --git a/actors/location/src/main/java/org/sunbird/location/actors/BaseLocationActor.java b/actors/location/src/main/java/org/sunbird/location/actors/BaseLocationActor.java new file mode 100644 index 000000000..dc6056474 --- /dev/null +++ b/actors/location/src/main/java/org/sunbird/location/actors/BaseLocationActor.java @@ -0,0 +1,86 @@ +package org.sunbird.location.actors; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.sunbird.actor.core.BaseActor; +import org.sunbird.common.models.util.GeoLocationJsonKey; +import org.sunbird.common.models.util.JsonKey; +import org.sunbird.common.models.util.ProjectLogger; +import org.sunbird.common.models.util.PropertiesCache; +import org.sunbird.common.request.Request; +import org.sunbird.dto.SearchDTO; +import org.sunbird.telemetry.util.TelemetryLmaxWriter; +import org.sunbird.telemetry.util.TelemetryUtil; + +/** @author Amit Kumar */ +public abstract class BaseLocationActor extends BaseActor { + + public void generateTelemetryForLocation( + String targetObjId, Map data, String operation) { + // object of telemetry event... + try { + Map targetObject = null; + List> correlatedObject = new ArrayList<>(); + targetObject = + TelemetryUtil.generateTargetObject(targetObjId, JsonKey.LOCATION, operation, null); + if (!MapUtils.isEmpty(data) + && StringUtils.isNotEmpty((String) data.get(GeoLocationJsonKey.PARENT_ID))) { + TelemetryUtil.generateCorrelatedObject( + (String) data.get(GeoLocationJsonKey.PARENT_ID), + JsonKey.LOCATION, + null, + correlatedObject); + } + TelemetryUtil.telemetryProcessingCall(data, targetObject, correlatedObject); + } catch (Exception e) { + ProjectLogger.log(e.getMessage(), e); + } + } + + public void generateSearchTelemetryEvent( + SearchDTO searchDto, String[] types, Map result) { + try { + Map telemetryContext = TelemetryUtil.getTelemetryContext(); + Map params = new HashMap<>(); + params.put(JsonKey.QUERY, searchDto.getQuery()); + params.put(JsonKey.FILTERS, searchDto.getAdditionalProperties().get(JsonKey.FILTERS)); + params.put(JsonKey.SORT, searchDto.getSortBy()); + params.put(JsonKey.TOPN, generateTopNResult(result)); + params.put(JsonKey.SIZE, result.get(JsonKey.COUNT)); + params.put(JsonKey.TYPE, String.join(",", types)); + + Request request = new Request(); + request.setRequest(telemetryRequestForSearch(telemetryContext, params)); + TelemetryLmaxWriter.getInstance().submitMessage(request); + } catch (Exception e) { + ProjectLogger.log(e.getMessage(), e); + } + } + + private List> generateTopNResult(Map result) { + List> dataMapList = (List>) result.get(JsonKey.CONTENT); + Integer topN = + Integer.parseInt(PropertiesCache.getInstance().getProperty(JsonKey.SEARCH_TOP_N)); + int count = Math.min(topN, dataMapList.size()); + List> list = new ArrayList<>(); + for (int i = 0; i < count; i++) { + Map m = new HashMap<>(); + m.put(JsonKey.ID, dataMapList.get(i).get(JsonKey.ID)); + list.add(m); + } + return list; + } + + private static Map telemetryRequestForSearch( + Map telemetryContext, Map params) { + Map map = new HashMap<>(); + map.put(JsonKey.CONTEXT, telemetryContext); + map.put(JsonKey.PARAMS, params); + map.put(JsonKey.TELEMETRY_EVENT_TYPE, "SEARCH"); + return map; + } +} diff --git a/actors/location/src/main/java/org/sunbird/location/actors/LocationActor.java b/actors/location/src/main/java/org/sunbird/location/actors/LocationActor.java new file mode 100644 index 000000000..7f50a47b5 --- /dev/null +++ b/actors/location/src/main/java/org/sunbird/location/actors/LocationActor.java @@ -0,0 +1,154 @@ +package org.sunbird.location.actors; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.HashMap; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.sunbird.actor.router.ActorConfig; +import org.sunbird.common.models.response.Response; +import org.sunbird.common.models.util.GeoLocationJsonKey; +import org.sunbird.common.models.util.JsonKey; +import org.sunbird.common.models.util.LocationActorOperation; +import org.sunbird.common.models.util.ProjectLogger; +import org.sunbird.common.models.util.ProjectUtil; +import org.sunbird.common.request.ExecutionContext; +import org.sunbird.common.request.Request; +import org.sunbird.dto.SearchDTO; +import org.sunbird.learner.util.Util; +import org.sunbird.location.dao.LocationDao; +import org.sunbird.location.dao.impl.LocationDaoFactory; +import org.sunbird.location.model.Location; +import org.sunbird.location.util.LocationRequestValidator; + +/** + * This class will handle all location related request. + * + * @author Amit Kumar + */ +@ActorConfig( + tasks = {"createLocation", "updateLocation", "searchLocation", "deleteLocation"}, + asyncTasks = {} +) +public class LocationActor extends BaseLocationActor { + + private ObjectMapper mapper = new ObjectMapper(); + private LocationDao locationDao = LocationDaoFactory.getInstance(); + + @Override + public void onReceive(Request request) throws Throwable { + Util.initializeContext(request, JsonKey.LOCATION); + ExecutionContext.setRequestId(request.getRequestId()); + String operation = request.getOperation(); + switch (operation) { + case "createLocation": + createLocation(request); + break; + case "updateLocation": + updateLocation(request); + break; + case "searchLocation": + searchLocation(request); + break; + case "deleteLocation": + deleteLocation(request); + break; + default: + onReceiveUnsupportedOperation("LocationActor"); + } + } + + private void createLocation(Request request) { + try { + Map data = request.getRequest(); + validateUpsertLocnReq(data, JsonKey.CREATE); + // put unique identifier in request for Id + String id = ProjectUtil.generateUniqueId(); + data.put(JsonKey.ID, id); + Location location = mapper.convertValue(data, Location.class); + Response response = locationDao.create(location); + sender().tell(response, self()); + ProjectLogger.log("Insert location data to ES"); + saveDataToES(data, JsonKey.INSERT); + generateTelemetryForLocation(id, data, JsonKey.CREATE); + } catch (Exception ex) { + ProjectLogger.log(ex.getMessage(), ex); + sender().tell(ex, self()); + } + } + + private void updateLocation(Request request) { + try { + Map data = request.getRequest(); + validateUpsertLocnReq(data, JsonKey.UPDATE); + Response response = locationDao.update(mapper.convertValue(data, Location.class)); + sender().tell(response, self()); + ProjectLogger.log("Update location data to ES"); + saveDataToES(data, JsonKey.UPDATE); + generateTelemetryForLocation((String) data.get(JsonKey.ID), data, JsonKey.UPDATE); + } catch (Exception ex) { + ProjectLogger.log(ex.getMessage(), ex); + sender().tell(ex, self()); + } + } + + private void searchLocation(Request request) { + try { + Response response = locationDao.search(request.getRequest()); + sender().tell(response, self()); + SearchDTO searchDto = Util.createSearchDto(request.getRequest()); + String[] types = {ProjectUtil.EsType.location.getTypeName()}; + generateSearchTelemetryEvent(searchDto, types, response.getResult()); + } catch (Exception ex) { + ProjectLogger.log(ex.getMessage(), ex); + sender().tell(ex, self()); + } + } + + private void deleteLocation(Request request) { + try { + String locationId = (String) request.getRequest().get(JsonKey.LOCATION_ID); + LocationRequestValidator.isLocationHasChild(locationId); + Response response = locationDao.delete(locationId); + sender().tell(response, self()); + ProjectLogger.log("Delete location data from ES"); + deleteDataFromES(locationId); + generateTelemetryForLocation(locationId, new HashMap<>(), JsonKey.DELETE); + } catch (Exception ex) { + ProjectLogger.log(ex.getMessage(), ex); + sender().tell(ex, self()); + } + } + + private void saveDataToES(Map locData, String opType) { + Request request = new Request(); + request.setOperation(LocationActorOperation.UPSERT_LOCATION_TO_ES.getValue()); + request.getRequest().put(JsonKey.LOCATION, locData); + request.getRequest().put(JsonKey.OPERATION_TYPE, opType); + try { + tellToAnother(request); + } catch (Exception ex) { + ProjectLogger.log("Exception Ocurred during saving location data to ES : ", ex); + } + } + + private void deleteDataFromES(String locId) { + Request request = new Request(); + request.setOperation(LocationActorOperation.DELETE_LOCATION_FROM_ES.getValue()); + request.getRequest().put(JsonKey.LOCATION_ID, locId); + try { + tellToAnother(request); + } catch (Exception ex) { + ProjectLogger.log("Exception Ocurred during saving location data to ES : ", ex); + } + } + + private void validateUpsertLocnReq(Map data, String operation) { + if (StringUtils.isNotEmpty((String) data.get(GeoLocationJsonKey.LOCATION_TYPE))) { + LocationRequestValidator.isValidLocationType( + (String) data.get(GeoLocationJsonKey.LOCATION_TYPE)); + } + LocationRequestValidator.isValidParentIdAndCode(data, operation); + // once parentCode validated remove from req as we are not saving this to our db + data.remove(GeoLocationJsonKey.PARENT_CODE); + } +} diff --git a/actors/location/src/main/java/org/sunbird/location/actors/LocationBackgroundActor.java b/actors/location/src/main/java/org/sunbird/location/actors/LocationBackgroundActor.java new file mode 100644 index 000000000..1d71d4f13 --- /dev/null +++ b/actors/location/src/main/java/org/sunbird/location/actors/LocationBackgroundActor.java @@ -0,0 +1,55 @@ +package org.sunbird.location.actors; + +import java.util.Map; +import org.sunbird.actor.router.ActorConfig; +import org.sunbird.common.ElasticSearchUtil; +import org.sunbird.common.models.util.JsonKey; +import org.sunbird.common.models.util.ProjectLogger; +import org.sunbird.common.models.util.ProjectUtil; +import org.sunbird.common.request.Request; + +/** + * This class will handle all background service for locationActor. + * + * @author Amit Kumar + */ +@ActorConfig( + tasks = {}, + asyncTasks = {"upsertLocationDataToES", "deleteLocationDataFromES"} +) +public class LocationBackgroundActor extends BaseLocationActor { + + @Override + public void onReceive(Request request) throws Throwable { + ProjectLogger.log("LocationBackgroundActor onReceive called"); + String operation = request.getOperation(); + + switch (operation) { + case "upsertLocationDataToES": + upsertLocationDataToES(request); + break; + case "deleteLocationDataFromES": + deleteLocationDataFromES(request); + break; + default: + onReceiveUnsupportedOperation("LocationBackgroundActor"); + } + } + + private void deleteLocationDataFromES(Request request) { + String locationId = (String) request.get(JsonKey.LOCATION_ID); + ElasticSearchUtil.removeData( + ProjectUtil.EsIndex.sunbird.getIndexName(), + ProjectUtil.EsType.location.getTypeName(), + locationId); + } + + private void upsertLocationDataToES(Request request) { + Map location = (Map) request.getRequest().get(JsonKey.LOCATION); + ElasticSearchUtil.upsertData( + ProjectUtil.EsIndex.sunbird.getIndexName(), + ProjectUtil.EsType.location.getTypeName(), + (String) location.get(JsonKey.ID), + location); + } +} diff --git a/actors/location/src/main/java/org/sunbird/location/dao/LocationDao.java b/actors/location/src/main/java/org/sunbird/location/dao/LocationDao.java new file mode 100644 index 000000000..87bec2140 --- /dev/null +++ b/actors/location/src/main/java/org/sunbird/location/dao/LocationDao.java @@ -0,0 +1,44 @@ +package org.sunbird.location.dao; + +import java.util.Map; +import org.sunbird.common.models.response.Response; +import org.sunbird.location.model.Location; + +/** @author Amit Kumar */ +public interface LocationDao { + /** + * @param location Location Details + * @return response Response + */ + Response create(Location location); + + /** + * @param location Location Details + * @return response Response + */ + Response update(Location location); + + /** + * @param locationId its a unique identity for Location + * @return response Response + */ + Response delete(String locationId); + + /** + * @param searchQueryMap Map it contains the filters to search Location from ES + * @return response Response + */ + Response search(Map searchQueryMap); + + /** + * @param locationId + * @return response Response + */ + Response read(String locationId); + + /** + * @param queryMap + * @return response Response + */ + Response getRecordByProperty(Map queryMap); +} diff --git a/actors/location/src/main/java/org/sunbird/location/dao/impl/LocationDaoFactory.java b/actors/location/src/main/java/org/sunbird/location/dao/impl/LocationDaoFactory.java new file mode 100644 index 000000000..88587488f --- /dev/null +++ b/actors/location/src/main/java/org/sunbird/location/dao/impl/LocationDaoFactory.java @@ -0,0 +1,25 @@ +package org.sunbird.location.dao.impl; + +import org.sunbird.location.dao.LocationDao; + +/** @author Amit Kumar */ +public class LocationDaoFactory { + + /** private default constructor. */ + private LocationDaoFactory() {} + + private static LocationDao locationDao; + + static { + locationDao = new LocationDaoImpl(); + } + + /** + * This method will provide singleton instance for LocationDaoImpl. + * + * @return LocationDao + */ + public static LocationDao getInstance() { + return locationDao; + } +} diff --git a/actors/location/src/main/java/org/sunbird/location/dao/impl/LocationDaoImpl.java b/actors/location/src/main/java/org/sunbird/location/dao/impl/LocationDaoImpl.java new file mode 100644 index 000000000..245c72166 --- /dev/null +++ b/actors/location/src/main/java/org/sunbird/location/dao/impl/LocationDaoImpl.java @@ -0,0 +1,77 @@ +package org.sunbird.location.dao.impl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.sunbird.cassandra.CassandraOperation; +import org.sunbird.common.ElasticSearchUtil; +import org.sunbird.common.models.response.Response; +import org.sunbird.common.models.util.GeoLocationJsonKey; +import org.sunbird.common.models.util.JsonKey; +import org.sunbird.common.models.util.ProjectUtil; +import org.sunbird.dto.SearchDTO; +import org.sunbird.helper.ServiceFactory; +import org.sunbird.learner.util.Util; +import org.sunbird.location.dao.LocationDao; +import org.sunbird.location.model.Location; + +/** @author Amit Kumar */ +public class LocationDaoImpl implements LocationDao { + + private CassandraOperation cassandraOperation = ServiceFactory.getInstance(); + private ObjectMapper mapper = new ObjectMapper(); + private static final String KEYSPACE_NAME = "sunbird"; + private static final String LOCATION_TABLE_NAME = "location"; + + @Override + public Response create(Location location) { + Map map = mapper.convertValue(location, Map.class); + Response response = cassandraOperation.insertRecord(KEYSPACE_NAME, LOCATION_TABLE_NAME, map); + // need to send ID along with success msg + response.put(JsonKey.ID, map.get(JsonKey.ID)); + return response; + } + + @Override + public Response update(Location location) { + Map map = mapper.convertValue(location, Map.class); + return cassandraOperation.updateRecord(KEYSPACE_NAME, LOCATION_TABLE_NAME, map); + } + + @Override + public Response delete(String locationId) { + return cassandraOperation.deleteRecord(KEYSPACE_NAME, LOCATION_TABLE_NAME, locationId); + } + + @Override + public Response search(Map searchQueryMap) { + SearchDTO searchDto = Util.createSearchDto(searchQueryMap); + String[] types = {ProjectUtil.EsType.location.getTypeName()}; + Map result = + ElasticSearchUtil.complexSearch( + searchDto, ProjectUtil.EsIndex.sunbird.getIndexName(), types); + Response response = new Response(); + if (result != null) { + response.put(JsonKey.RESPONSE, result.get(JsonKey.CONTENT)); + } else { + List> list = new ArrayList<>(); + response.put(JsonKey.RESPONSE, list); + } + return response; + } + + @Override + public Response read(String locationId) { + return cassandraOperation.getRecordById(KEYSPACE_NAME, LOCATION_TABLE_NAME, locationId); + } + + @Override + public Response getRecordByProperty(Map queryMap) { + return cassandraOperation.getRecordsByProperty( + KEYSPACE_NAME, + LOCATION_TABLE_NAME, + (String) queryMap.get(GeoLocationJsonKey.PROPERTY_NAME), + queryMap.get(GeoLocationJsonKey.PROPERTY_VALUE)); + } +} diff --git a/actors/location/src/main/java/org/sunbird/location/model/Location.java b/actors/location/src/main/java/org/sunbird/location/model/Location.java new file mode 100644 index 000000000..d128d601d --- /dev/null +++ b/actors/location/src/main/java/org/sunbird/location/model/Location.java @@ -0,0 +1,60 @@ +package org.sunbird.location.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import java.io.Serializable; + +/** @author Amit Kumar */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(Include.NON_NULL) +public class Location implements Serializable { + + private static final long serialVersionUID = -7967252522327069670L; + + private String id; + private String code; + private String name; + private String type; + private String parentId; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getParentId() { + return parentId; + } + + public void setParentId(String parentId) { + this.parentId = parentId; + } +} diff --git a/actors/location/src/main/java/org/sunbird/location/util/LocationRequestValidator.java b/actors/location/src/main/java/org/sunbird/location/util/LocationRequestValidator.java new file mode 100644 index 000000000..127c99b6e --- /dev/null +++ b/actors/location/src/main/java/org/sunbird/location/util/LocationRequestValidator.java @@ -0,0 +1,339 @@ +package org.sunbird.location.util; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.sunbird.common.ElasticSearchUtil; +import org.sunbird.common.exception.ProjectCommonException; +import org.sunbird.common.models.response.Response; +import org.sunbird.common.models.util.GeoLocationJsonKey; +import org.sunbird.common.models.util.JsonKey; +import org.sunbird.common.models.util.ProjectUtil; +import org.sunbird.common.responsecode.ResponseCode; +import org.sunbird.dto.SearchDTO; +import org.sunbird.learner.util.Util; +import org.sunbird.location.dao.LocationDao; +import org.sunbird.location.dao.impl.LocationDaoImpl; + +/** @author Amit Kumar */ +public class LocationRequestValidator { + + private LocationRequestValidator() {} + + private static LocationDao locationDao = new LocationDaoImpl(); + protected static List> locationTypeGroupList = new ArrayList<>(); + protected static List typeList = new ArrayList<>(); + + static { + List subTypeList = + Arrays.asList( + ProjectUtil.getConfigValue(GeoLocationJsonKey.SUNBIRD_VALID_LOCATION_TYPES).split(";")); + for (String str : subTypeList) { + typeList.addAll( + ((Arrays.asList(str.split(","))) + .stream() + .map( + x -> { + return x.toLowerCase(); + })) + .collect(Collectors.toList())); + locationTypeGroupList.add( + ((Arrays.asList(str.split(","))) + .stream() + .map( + x -> { + return x.toLowerCase(); + })) + .collect(Collectors.toList())); + } + } + + /** + * This method will validate location code + * + * @param code + * @return boolean + */ + public static boolean isValidLocationCode(String code) { + Map reqMap = new HashMap<>(); + reqMap.put(GeoLocationJsonKey.PROPERTY_NAME, GeoLocationJsonKey.CODE); + reqMap.put(GeoLocationJsonKey.PROPERTY_VALUE, code); + Response response = locationDao.getRecordByProperty(reqMap); + List> locationMapList = + (List>) response.get(JsonKey.RESPONSE); + return (!locationMapList.isEmpty()); + } + + /** + * This method will validate location type + * + * @param type + * @return + */ + public static boolean isValidLocationType(String type) { + if (null != type && !typeList.contains(type.toLowerCase())) { + throw new ProjectCommonException( + ResponseCode.invalidValue.getErrorCode(), + ProjectUtil.formatMessage( + ResponseCode.invalidValue.getErrorMessage(), + GeoLocationJsonKey.LOCATION_TYPE, + type, + typeList), + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + return true; + } + + /** + * This method will validate parentId , parentCode and locationType from a given request + * + * @param location + * @param opType + * @return boolean + */ + public static boolean isValidParentIdAndCode(Map location, String opType) { + String type = (String) location.get(GeoLocationJsonKey.LOCATION_TYPE); + if (StringUtils.isNotEmpty(type)) { + List locationTypeList = getLocationSubTypeListForType(type); + // if type is of top level then no need to validate parentCode and parentId + if (!locationTypeList.get(0).equalsIgnoreCase(type.toLowerCase())) { + // while creating new location, if locationType is not top level then type and parent id is + // mandatory + if ((StringUtils.isEmpty((String) location.get(GeoLocationJsonKey.PARENT_CODE)) + && StringUtils.isEmpty((String) location.get(GeoLocationJsonKey.PARENT_ID)))) { + throw new ProjectCommonException( + ResponseCode.mandatoryParamsMissing.getErrorCode(), + ProjectUtil.formatMessage( + ResponseCode.mandatoryParamsMissing.getErrorMessage(), + (GeoLocationJsonKey.PARENT_ID + " or " + GeoLocationJsonKey.PARENT_CODE)), + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + } else if (locationTypeList.get(0).equalsIgnoreCase(type.toLowerCase())) { + if (StringUtils.isNotEmpty((String) location.get(GeoLocationJsonKey.PARENT_CODE)) + || StringUtils.isNotEmpty((String) location.get(GeoLocationJsonKey.PARENT_ID))) { + throw new ProjectCommonException( + ResponseCode.parentNotAllowed.getErrorCode(), + ProjectUtil.formatMessage( + ResponseCode.parentNotAllowed.getErrorMessage(), + (GeoLocationJsonKey.PARENT_ID + " or " + GeoLocationJsonKey.PARENT_CODE)), + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + // if type is top level then parentCode and parentId is null + location.put(GeoLocationJsonKey.PARENT_CODE, null); + location.put(GeoLocationJsonKey.PARENT_ID, null); + } + } + if (StringUtils.isNotEmpty((String) location.get(GeoLocationJsonKey.CODE))) { + isValidLocationCode(location, opType); + } + validateParentIDAndParentCode(location, opType); + return true; + } + + private static List getLocationSubTypeListForType(String type) { + for (List subList : locationTypeGroupList) { + if (subList.contains(type.toLowerCase())) { + return subList; + } + } + return (new ArrayList<>()); + } + + private static void validateParentIDAndParentCode(Map location, String opType) { + String parentCode = (String) location.get(GeoLocationJsonKey.PARENT_CODE); + String parentId = (String) location.get(GeoLocationJsonKey.PARENT_ID); + if (StringUtils.isNotEmpty(parentCode)) { + Map map = getLocation(parentCode); + parentId = (String) map.get(JsonKey.ID); + location.put(GeoLocationJsonKey.PARENT_ID, map.get(JsonKey.ID)); + } + if (StringUtils.isNotEmpty(parentId)) { + String operation = GeoLocationJsonKey.PARENT_ID; + if (StringUtils.isNotEmpty(parentCode)) { + operation = GeoLocationJsonKey.PARENT_CODE; + } + Map parentLocation = getLocationById(parentId, operation); + validateParentLocationType(parentLocation, location, opType); + } + } + + /** + * This method will validate the parent location type means parent should be only one level up + * from child + * + * @param parentLocation + * @param location + * @return + */ + private static boolean validateParentLocationType( + Map parentLocation, Map location, String opType) { + String parentType = (String) parentLocation.get(GeoLocationJsonKey.LOCATION_TYPE); + String currentLocType = (String) location.get(GeoLocationJsonKey.LOCATION_TYPE); + Map locn = null; + if (opType.equalsIgnoreCase(JsonKey.UPDATE)) { + locn = getLocationById((String) location.get(JsonKey.ID), JsonKey.LOCATION_ID); + currentLocType = (String) locn.get(GeoLocationJsonKey.LOCATION_TYPE); + } + Map currentLocTypeoOrdermap = + getLocationTypeOrderMap(currentLocType.toLowerCase()); + Map parentLocTypeoOrdermap = + getLocationTypeOrderMap(currentLocType.toLowerCase()); + if ((currentLocTypeoOrdermap.get(currentLocType.toLowerCase()) + - parentLocTypeoOrdermap.get(parentType.toLowerCase())) + != 1) { + throw new ProjectCommonException( + ResponseCode.invalidParameter.getErrorCode(), + ProjectUtil.formatMessage( + ResponseCode.invalidParameter.getErrorMessage(), + (GeoLocationJsonKey.PARENT_ID + " or " + GeoLocationJsonKey.PARENT_CODE)), + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + return false; + } + + private static Map getLocationTypeOrderMap(String type) { + List list = getLocationSubTypeListForType(type); + Map locTypeOrderMap = new LinkedHashMap<>(); + for (int i = 0; i < list.size(); i++) { + locTypeOrderMap.put(list.get(i), i); + } + return locTypeOrderMap; + } + + /** + * This method will return location details based on id + * + * @param id + * @return Map location details + */ + private static Map getLocationById(String id, String parameter) { + Map location = + ElasticSearchUtil.getDataByIdentifier( + ProjectUtil.EsIndex.sunbird.getIndexName(), + ProjectUtil.EsType.location.getTypeName(), + id); + if (MapUtils.isEmpty(location)) { + throw new ProjectCommonException( + ResponseCode.invalidParameter.getErrorCode(), + ProjectUtil.formatMessage(ResponseCode.invalidParameter.getErrorMessage(), parameter), + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + return location; + } + + /** + * This method will validate location code and if code is valid will return location . + * + * @param location Map complete location details + * @param opType (for which operation we are validating i.e CREATE or UPDATE) + * @return location details Map + */ + private static Map getLocation(String parentCode) { + Map filters = new HashMap<>(); + filters.put(GeoLocationJsonKey.CODE, parentCode); + Map map = new HashMap<>(); + map.put(JsonKey.FILTERS, filters); + List> locationMapList = + getESSearchResult( + map, + ProjectUtil.EsIndex.sunbird.getIndexName(), + ProjectUtil.EsType.location.getTypeName()); + if (CollectionUtils.isNotEmpty(locationMapList)) { + return locationMapList.get(0); + } else { + throw new ProjectCommonException( + ResponseCode.invalidParameter.getErrorCode(), + ProjectUtil.formatMessage( + ResponseCode.invalidParameter.getErrorMessage(), GeoLocationJsonKey.PARENT_CODE), + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + } + + /** + * This method will check whether passed locationId is the parent of any location + * + * @param locationId + * @return boolean + */ + public static boolean isLocationHasChild(String locationId) { + Map location = getLocationById(locationId, JsonKey.LOCATION_ID); + Map locTypeoOrdermap = + getLocationTypeOrderMap( + ((String) location.get(GeoLocationJsonKey.LOCATION_TYPE)).toLowerCase()); + List list = new ArrayList<>(locTypeoOrdermap.values()); + list.sort(Comparator.reverseOrder()); + int order = + locTypeoOrdermap.get( + ((String) location.get(GeoLocationJsonKey.LOCATION_TYPE)).toLowerCase()); + // location type with last order can be deleted without validation + if (order != list.get(0)) { + Map filters = new HashMap<>(); + filters.put(GeoLocationJsonKey.PARENT_ID, location.get(JsonKey.ID)); + Map map = new HashMap<>(); + map.put(JsonKey.FILTERS, filters); + List> locationMapList = + getESSearchResult( + map, + ProjectUtil.EsIndex.sunbird.getIndexName(), + ProjectUtil.EsType.location.getTypeName()); + if (CollectionUtils.isNotEmpty(locationMapList)) { + throw new ProjectCommonException( + ResponseCode.invalidLocationDeleteRequest.getErrorCode(), + ResponseCode.invalidLocationDeleteRequest.getErrorMessage(), + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + } + return true; + } + + public static List> getESSearchResult( + Map searchQueryMap, String esIndex, String esType) { + SearchDTO searchDto = Util.createSearchDto(searchQueryMap); + Map result = ElasticSearchUtil.complexSearch(searchDto, esIndex, esType); + return (List>) result.get(JsonKey.CONTENT); + } + + public static boolean isValidLocationCode(Map location, String opType) { + Map filters = new HashMap<>(); + filters.put(GeoLocationJsonKey.CODE, location.get(GeoLocationJsonKey.CODE)); + Map map = new HashMap<>(); + map.put(JsonKey.FILTERS, filters); + List> locationMapList = + getESSearchResult( + map, + ProjectUtil.EsIndex.sunbird.getIndexName(), + ProjectUtil.EsType.location.getTypeName()); + if (!locationMapList.isEmpty()) { + if (opType.equalsIgnoreCase(JsonKey.CREATE)) { + throw new ProjectCommonException( + ResponseCode.alreadyExists.getErrorCode(), + ProjectUtil.formatMessage( + ResponseCode.alreadyExists.getErrorMessage(), + GeoLocationJsonKey.CODE, + location.get(GeoLocationJsonKey.CODE)), + ResponseCode.CLIENT_ERROR.getResponseCode()); + } else if (opType.equalsIgnoreCase(JsonKey.UPDATE)) { + Map locn = locationMapList.get(0); + if (!(((String) locn.get(JsonKey.ID)) + .equalsIgnoreCase((String) location.get(JsonKey.ID)))) { + throw new ProjectCommonException( + ResponseCode.alreadyExists.getErrorCode(), + ProjectUtil.formatMessage( + ResponseCode.alreadyExists.getErrorMessage(), + GeoLocationJsonKey.CODE, + location.get(GeoLocationJsonKey.CODE)), + ResponseCode.CLIENT_ERROR.getResponseCode()); + } + } + } + return true; + } +} diff --git a/actors/location/src/test/java/org/sunbird/location/actors/LocationActorTest.java b/actors/location/src/test/java/org/sunbird/location/actors/LocationActorTest.java new file mode 100644 index 000000000..979d6a387 --- /dev/null +++ b/actors/location/src/test/java/org/sunbird/location/actors/LocationActorTest.java @@ -0,0 +1,137 @@ +package org.sunbird.location.actors; + +import static akka.testkit.JavaTestKit.duration; + +import akka.actor.ActorRef; +import akka.actor.ActorSystem; +import akka.actor.Props; +import akka.testkit.javadsl.TestKit; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.sunbird.common.ElasticSearchUtil; +import org.sunbird.common.models.response.Response; +import org.sunbird.common.models.util.GeoLocationJsonKey; +import org.sunbird.common.models.util.JsonKey; +import org.sunbird.common.models.util.LocationActorOperation; +import org.sunbird.common.request.Request; +import org.sunbird.location.dao.LocationDao; +import org.sunbird.location.dao.impl.LocationDaoFactory; +import org.sunbird.location.dao.impl.LocationDaoImpl; +import org.sunbird.location.model.Location; +import scala.concurrent.duration.FiniteDuration; + +@Ignore +@RunWith(PowerMockRunner.class) +@PrepareForTest({LocationDaoFactory.class, LocationDaoImpl.class, ElasticSearchUtil.class}) +@PowerMockIgnore({"javax.management.*", "javax.net.ssl.*", "javax.security.*"}) +public class LocationActorTest { + + private static final FiniteDuration ACTOR_MAX_WAIT_DURATION = duration("100 second"); + private ObjectMapper mapper = new ObjectMapper(); + private ActorSystem system; + private Props props; + private LocationDao locDaoImpl; + private TestKit probe; + private ActorRef subject; + // private LocationDaoFactory locFactory; + private Request actorMessage; + + @Before + public void setUp() { + system = ActorSystem.create("system"); + probe = new TestKit(system); + PowerMockito.mockStatic(LocationDaoFactory.class); + PowerMockito.mockStatic(ElasticSearchUtil.class); + locDaoImpl = PowerMockito.mock(LocationDaoImpl.class); + props = Props.create(LocationActor.class); + subject = system.actorOf(props); + actorMessage = new Request(); + } + + @Test + public void testCreateLocation() throws IOException { + Response response = new Response(); + response.put(JsonKey.RESPONSE, JsonKey.SUCCESS); + response.put(JsonKey.RESPONSE, response); + Map data = new HashMap(); + data.put(GeoLocationJsonKey.LOCATION_TYPE, "STATE"); + data.put(GeoLocationJsonKey.CODE, "S01"); + data.put(JsonKey.NAME, "DUMMY_STATE"); + Location location = mapper.convertValue(data, Location.class); + PowerMockito.when(LocationDaoFactory.getInstance()).thenReturn(locDaoImpl); + PowerMockito.when(locDaoImpl.create(location)).thenReturn(response); + actorMessage.setOperation(LocationActorOperation.CREATE_LOCATION.getValue()); + actorMessage.getRequest().putAll(data); + subject.tell(actorMessage, probe.getRef()); + Response resp = probe.expectMsgClass(ACTOR_MAX_WAIT_DURATION, Response.class); + Assert.assertTrue(null != resp); + } + + @Test + public void testUpdateLocation() throws IOException { + Response response = new Response(); + response.put(JsonKey.RESPONSE, JsonKey.SUCCESS); + response.put(JsonKey.RESPONSE, response); + Map data = new HashMap(); + data.put(GeoLocationJsonKey.LOCATION_TYPE, "STATE"); + data.put(GeoLocationJsonKey.CODE, "S01"); + data.put(JsonKey.NAME, "DUMMY_STATE_CHANGE"); + data.put(JsonKey.ID, "id_01"); + Location location = mapper.convertValue(data, Location.class); + PowerMockito.when(LocationDaoFactory.getInstance()).thenReturn(locDaoImpl); + PowerMockito.when(locDaoImpl.create(location)).thenReturn(response); + actorMessage.setOperation(LocationActorOperation.UPDATE_LOCATION.getValue()); + actorMessage.getRequest().putAll(data); + subject.tell(actorMessage, probe.getRef()); + Response resp = probe.expectMsgClass(ACTOR_MAX_WAIT_DURATION, Response.class); + Assert.assertTrue(null != resp); + } + + @Test + public void testDeleteLocation() throws IOException { + Response response = new Response(); + response.put(JsonKey.RESPONSE, JsonKey.SUCCESS); + response.put(JsonKey.RESPONSE, response); + Map data = new HashMap(); + data.put(JsonKey.ID, "id_01"); + Location location = mapper.convertValue(data, Location.class); + PowerMockito.when(LocationDaoFactory.getInstance()).thenReturn(locDaoImpl); + PowerMockito.when(locDaoImpl.create(location)).thenReturn(response); + actorMessage.setOperation(LocationActorOperation.DELETE_LOCATION.getValue()); + actorMessage.getRequest().putAll(data); + subject.tell(actorMessage, probe.getRef()); + Response resp = probe.expectMsgClass(ACTOR_MAX_WAIT_DURATION, Response.class); + Assert.assertTrue(null != resp); + } + + @Test + public void testSearchLocation() throws IOException { + Response response = new Response(); + response.put(JsonKey.RESPONSE, JsonKey.SUCCESS); + response.put(JsonKey.RESPONSE, response); + Map data = new HashMap(); + data.put(GeoLocationJsonKey.LOCATION_TYPE, "STATE"); + data.put(GeoLocationJsonKey.CODE, "S01"); + data.put(JsonKey.NAME, "DUMMY_STATE_CHANGE"); + data.put(JsonKey.ID, "id_01"); + Location location = mapper.convertValue(data, Location.class); + PowerMockito.when(LocationDaoFactory.getInstance()).thenReturn(locDaoImpl); + PowerMockito.when(locDaoImpl.create(location)).thenReturn(response); + actorMessage.setOperation(LocationActorOperation.SEARCH_LOCATION.getValue()); + actorMessage.getRequest().put(JsonKey.FILTERS, data); + subject.tell(actorMessage, probe.getRef()); + Response resp = probe.expectMsgClass(ACTOR_MAX_WAIT_DURATION, Response.class); + Assert.assertTrue(null != resp); + } +} diff --git a/actors/pom.xml b/actors/pom.xml index fd4418041..ebe51431b 100644 --- a/actors/pom.xml +++ b/actors/pom.xml @@ -19,5 +19,6 @@ content badge dashboard + location diff --git a/service/pom.xml b/service/pom.xml index 7ca1c4ff2..2227c8ca1 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -49,6 +49,11 @@ badge 1.0-SNAPSHOT + + org.sunbird + location + 1.0-SNAPSHOT + com.typesafe.akka akka-testkit_2.11 diff --git a/service/src/main/resources/application.conf b/service/src/main/resources/application.conf index 38d484242..689c9c5b7 100644 --- a/service/src/main/resources/application.conf +++ b/service/src/main/resources/application.conf @@ -254,7 +254,27 @@ SunbirdMWSystem { { router = smallest-mailbox-pool nr-of-instances = 4 - } + } + "/RequestRouter/*/LocationActor" + { + router = smallest-mailbox-pool + nr-of-instances = 2 + } + "/BackgroundRequestRouter/*/LocationBackgroundActor" + { + router = smallest-mailbox-pool + nr-of-instances = 2 + } + "/RequestRouter/*/LocationBulkUploadActor" + { + router = smallest-mailbox-pool + nr-of-instances = 2 + } + "/BackgroundRequestRouter/*/LocationBulkUploadBackGroundJobActor" + { + router = smallest-mailbox-pool + nr-of-instances = 2 + } } } remote { diff --git a/service/src/test/java/org/sunbird/learner/actors/bulkupload/LocationBulkUploadActorTest.java b/service/src/test/java/org/sunbird/learner/actors/bulkupload/LocationBulkUploadActorTest.java new file mode 100644 index 000000000..2cabbe0bf --- /dev/null +++ b/service/src/test/java/org/sunbird/learner/actors/bulkupload/LocationBulkUploadActorTest.java @@ -0,0 +1,165 @@ +package org.sunbird.learner.actors.bulkupload; + +import static akka.testkit.JavaTestKit.duration; + +import akka.actor.ActorRef; +import akka.actor.ActorSystem; +import akka.actor.Props; +import akka.testkit.javadsl.TestKit; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runners.MethodSorters; +import org.sunbird.actor.service.SunbirdMWService; +import org.sunbird.common.exception.ProjectCommonException; +import org.sunbird.common.models.response.Response; +import org.sunbird.common.models.util.BulkUploadActorOperation; +import org.sunbird.common.models.util.GeoLocationJsonKey; +import org.sunbird.common.models.util.JsonKey; +import org.sunbird.common.request.Request; +import org.sunbird.learner.util.Util; + +/** + * Test case for Location Bulk upload. + * + * @author arvind on 30/4/18. + */ +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class LocationBulkUploadActorTest { + + private static ActorSystem system; + private static final Props props = Props.create(LocationBulkUploadActor.class); + private static final String USER_ID = "user123"; + private static final String LOCATION_TYPE = "State"; + private ObjectMapper mapper = new ObjectMapper(); + + @BeforeClass + public static void setUp() { + SunbirdMWService.init(); + system = ActorSystem.create("system"); + Util.checkCassandraDbConnections(JsonKey.SUNBIRD); + } + + @Test + public void testLocationBulkUploadWithProperData() throws Exception { + TestKit probe = new TestKit(system); + ActorRef subject = system.actorOf(props); + List headerLine = + Arrays.asList( + GeoLocationJsonKey.PROPERTY_NAME, + GeoLocationJsonKey.CODE, + GeoLocationJsonKey.PARENT_CODE, + GeoLocationJsonKey.PARENT_ID); + List firstDataLine = Arrays.asList("location_name", "location-code", null, null); + String jsonString = createLines(headerLine, firstDataLine); + Request reqObj = getRequestObjectForLocationBulkUpload(LOCATION_TYPE, jsonString.getBytes()); + subject.tell(reqObj, probe.getRef()); + Response res = probe.expectMsgClass(duration("100 second"), Response.class); + String processId = (String) res.get(JsonKey.ID); + Assert.assertTrue(null != processId); + } + + @Test + public void testLocationBulkUploadWithInvalidAttributeNames() throws Exception { + TestKit probe = new TestKit(system); + ActorRef subject = system.actorOf(props); + List headerLine = + Arrays.asList( + GeoLocationJsonKey.PROPERTY_NAME + "invalid", + GeoLocationJsonKey.CODE, + GeoLocationJsonKey.PARENT_CODE, + GeoLocationJsonKey.PARENT_ID); + List firstDataLine = Arrays.asList("location_name", "location-code", null, null); + String jsonString = createLines(headerLine, firstDataLine); + Request reqObj = getRequestObjectForLocationBulkUpload(LOCATION_TYPE, jsonString.getBytes()); + subject.tell(reqObj, probe.getRef()); + ProjectCommonException res = + probe.expectMsgClass(duration("10 second"), ProjectCommonException.class); + Assert.assertTrue(null != res); + } + + @Test + public void testLocationBulkUploadWithoutMandatoryFieldCode() throws Exception { + TestKit probe = new TestKit(system); + ActorRef subject = system.actorOf(props); + List headerLine = + Arrays.asList( + GeoLocationJsonKey.PROPERTY_NAME, + GeoLocationJsonKey.PARENT_CODE, + GeoLocationJsonKey.PARENT_ID); + List firstDataLine = Arrays.asList("location_name", null, null); + String jsonString = createLines(headerLine, firstDataLine); + Request reqObj = getRequestObjectForLocationBulkUpload(LOCATION_TYPE, jsonString.getBytes()); + subject.tell(reqObj, probe.getRef()); + ProjectCommonException res = + probe.expectMsgClass(duration("10 second"), ProjectCommonException.class); + Assert.assertTrue(null != res); + } + + @Test + public void testLocationBulkUploadWithoutAnyDataRecord() throws Exception { + TestKit probe = new TestKit(system); + ActorRef subject = system.actorOf(props); + List headerLine = + Arrays.asList( + GeoLocationJsonKey.PROPERTY_NAME, + GeoLocationJsonKey.PARENT_CODE, + GeoLocationJsonKey.PARENT_ID); + String jsonString = createLines(headerLine); + Request reqObj = getRequestObjectForLocationBulkUpload(LOCATION_TYPE, jsonString.getBytes()); + subject.tell(reqObj, probe.getRef()); + ProjectCommonException res = + probe.expectMsgClass(duration("10 second"), ProjectCommonException.class); + Assert.assertTrue(null != res); + } + + @Test + public void testLocationBulkUploadWithExtraAttributeNameValue() throws Exception { + TestKit probe = new TestKit(system); + ActorRef subject = system.actorOf(props); + List headerLine = + Arrays.asList( + GeoLocationJsonKey.PROPERTY_NAME, + GeoLocationJsonKey.CODE, + GeoLocationJsonKey.PARENT_CODE, + GeoLocationJsonKey.PARENT_ID, + GeoLocationJsonKey.PROPERTY_VALUE); + List firstDataLine = + Arrays.asList("location_name", "location-code", null, null, "value"); + String jsonString = createLines(headerLine, firstDataLine); + Request reqObj = getRequestObjectForLocationBulkUpload(LOCATION_TYPE, jsonString.getBytes()); + subject.tell(reqObj, probe.getRef()); + ProjectCommonException res = + probe.expectMsgClass(duration("10 second"), ProjectCommonException.class); + Assert.assertTrue(null != res); + } + + private Request getRequestObjectForLocationBulkUpload(String locationType, byte[] file) { + Request reqObj = new Request(); + reqObj.setOperation(BulkUploadActorOperation.LOCATION_BULK_UPLOAD.getValue()); + HashMap innerMap = new HashMap<>(); + innerMap.put(JsonKey.CREATED_BY, USER_ID); + innerMap.put(JsonKey.OBJECT_TYPE, JsonKey.LOCATION); + innerMap.put(JsonKey.FILE, file); + innerMap.put(GeoLocationJsonKey.LOCATION_TYPE, locationType); + reqObj.getRequest().put(JsonKey.DATA, innerMap); + return reqObj; + } + + private String createLines(List... list) throws JsonProcessingException { + + StringBuilder builder = new StringBuilder(); + for (List l : list) { + String.join(",", l); + builder.append(String.join(",", l)); + builder.append(System.lineSeparator()); + } + return builder.toString(); + } +}