Merge pull request #271 from Open-MBEE/release/4.0.20

Release/4.0.20

.circleci/config.yml

@@ -20,19 +20,21 @@ jobs:
 
       - setup_remote_docker
 
-      - run: 
+      - run:
           name: "Create and start all services from the docker-compose configuration"
           command: |
             cp example/src/main/resources/application.properties.example ./example/src/main/resources/application.properties
             docker-compose up --build -d
             docker run --network container:mms curlimages/curl --retry 8 --retry-delay 10 --retry-max-time 90 --retry-connrefused http://mms:8080/healthcheck
 
-      - run: 
+      - run:
           name: "Run and test Postman Collection"
           command: |
             docker create -v /etc/newman --name mms_test_configs alpine:3.4 /bin/true
             docker cp example/. mms_test_configs:/etc/newman
             docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run crud.postman_collection.json -e test-env.json --delay-request 500
+            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run getAtCommits.postman_collection.json -e test-env.json --delay-request 500
+            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run makeBranchFromCommit.postman_collection.json -e test-env.json --delay-request 500
             docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run cameo.postman_collection.json -e test-env.json --delay-request 1000
             docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run jupyter.postman_collection.json -e test-env.json --delay-request 500
             docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run localauth.postman_collection.json -e test-env.json --delay-request 500

.readthedocs.yaml

@@ -17,6 +17,6 @@ sphinx:
 
 # We recommend specifying your dependencies to enable reproducible builds:
 # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
-# python:
-#   install:
-#   - requirements: docs/requirements.txt
+python:
+  install:
+    - requirements: docs/requirements.txt

build.gradle

@@ -62,6 +62,7 @@ subprojects {
 
     group = "org.openmbee.mms"
     version = rootProject.version
+    ext['jackson-bom.version'] = jacksonVersion
 
     Map<String, String> commonDependencies = rootProject.ext.commonDependencies
 
@@ -157,4 +158,4 @@ subprojects {
             sign publishing.publications.mavenJava
         }
     }
-}
+}

core/src/main/java/org/openmbee/mms/core/dao/CommitIndexDAO.java

@@ -25,6 +25,8 @@ public interface CommitIndexDAO {
 
     List<CommitJson> elementHistory(String id, Set<String> commitIds);
 
+    List<CommitJson> elementDeletedHistory(String id, Collection<String> commitIds);
+
     CommitJson update(CommitJson commitJson);
 
 }

core/src/main/java/org/openmbee/mms/core/services/BranchService.java

@@ -10,6 +10,7 @@ public interface BranchService {
     RefsResponse getBranch(String projectId, String id);
 
     RefJson createBranch(String projectId, RefJson branch);
+    RefJson createBranchfromCommit(String projectId, RefJson branch, NodeService nodeService);
 
     RefsResponse deleteBranch(String projectId, String id);
 }

crud/src/main/java/org/openmbee/mms/crud/controllers/branches/BranchesController.java

@@ -96,9 +96,7 @@ public RefsResponse createRefs(
                 if (branch.getParentCommitId() == null || branch.getParentCommitId().isEmpty()) {
                     res = branchService.createBranch(projectId, branch);
                 } else {
-                    //TODO implement branching from historical commit
-                    response.addRejection(new Rejection(branch, 400, "Branching from historical commits is not implemented."));
-                    continue;
+                    res = branchService.createBranchfromCommit(projectId, branch, getNodeService(projectId));
                 }
 
                 permissionService.initBranchPerms(projectId, branch.getId(), true, auth.getName());

crud/src/main/java/org/openmbee/mms/crud/services/DefaultBranchService.java

@@ -5,11 +5,7 @@
 import org.openmbee.mms.core.config.Constants;
 import org.openmbee.mms.core.config.ContextHolder;
 import org.openmbee.mms.core.config.Formats;
-import org.openmbee.mms.core.dao.BranchDAO;
-import org.openmbee.mms.core.dao.BranchIndexDAO;
-import org.openmbee.mms.core.dao.CommitDAO;
-import org.openmbee.mms.core.dao.NodeDAO;
-import org.openmbee.mms.core.dao.NodeIndexDAO;
+import org.openmbee.mms.core.dao.*;
 import org.openmbee.mms.core.exceptions.BadRequestException;
 import org.openmbee.mms.core.exceptions.DeletedException;
 import org.openmbee.mms.core.exceptions.InternalErrorException;
@@ -18,9 +14,11 @@
 import org.openmbee.mms.core.objects.RefsResponse;
 import org.openmbee.mms.core.services.BranchService;
 import org.openmbee.mms.core.services.EventService;
+import org.openmbee.mms.core.services.NodeService;
 import org.openmbee.mms.data.domains.scoped.Branch;
 import org.openmbee.mms.data.domains.scoped.Commit;
 import org.openmbee.mms.data.domains.scoped.Node;
+import org.openmbee.mms.json.ElementJson;
 import org.openmbee.mms.json.RefJson;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -45,6 +43,15 @@ public class DefaultBranchService implements BranchService {
     private NodeIndexDAO nodeIndex;
 
     protected Collection<EventService> eventPublisher;
+    protected NodeGetHelper nodeGetHelper;
+
+
+
+    @Autowired
+    public void setNodeGetHelper(NodeGetHelper nodeGetHelper) {
+        this.nodeGetHelper = nodeGetHelper;
+    }
+
 
     @Autowired
     public void setBranchRepository(BranchDAO branchRepository) {
@@ -129,8 +136,8 @@ public RefJson createBranch(String projectId, RefJson branch) {
         branch.setCreated(Formats.FORMATTER.format(now));
         branch.setDeleted(false);
         branch.setProjectId(projectId);
-        branch.setStatus("created");
-
+        boolean fromCommit = branch.getParentCommitId() == null ? false : true;
+        branch.setStatus(fromCommit ? "creating" : "created");
         if (branch.getDocId() == null || branch.getDocId().isEmpty()) {
             String docId = branchIndex.createDocId(branch);
             branch.setDocId(docId);
@@ -145,18 +152,26 @@ public RefJson createBranch(String projectId, RefJson branch) {
             b.setParentRefId(Constants.MASTER_BRANCH);
         }
 
-        //This service cannot create branches from historic versions
-        if (branch.getParentCommitId() != null) {
-            throw new BadRequestException("Internal Error: Invalid branch creation logic.");
-        }
-
         Optional<Branch> refOption = branchRepository.findByBranchId(b.getParentRefId());
         if (refOption.isPresent()) {
-            Optional<Commit> parentCommit = commitRepository.findLatestByRef(refOption.get());
-            parentCommit.ifPresent(parent -> {
-                b.setParentCommit(parent.getId());
-                branch.setParentCommitId(parent.getCommitId()); //commit id is same as its docId
-            });
+            if(branch.getParentCommitId() != null){
+                Optional<Commit> commitRequestId = commitRepository.findByCommitId(branch.getParentCommitId());
+                commitRequestId.ifPresentOrElse(commit -> {
+                    Optional<Commit> parentCommit = commitRepository.findByRefAndTimestamp(refOption.get(), commit.getTimestamp());
+                    parentCommit.ifPresent(parent -> {
+                        b.setParentCommit(parent.getId());
+                        branch.setParentCommitId(parent.getCommitId());
+                    });
+                },
+                    () -> { throw new BadRequestException(new RefsResponse().addMessage("parentCommitId not found " + now.toString()));
+                });
+            } else {
+                Optional<Commit> parentCommit = commitRepository.findLatestByRef(refOption.get());
+                parentCommit.ifPresent(parent -> {
+                    b.setParentCommit(parent.getId());
+                    branch.setParentCommitId(parent.getCommitId()); //commit id is same as its docId
+                });
+            }
         }
 
         if (b.getParentCommit() == null) {
@@ -167,11 +182,14 @@ public RefJson createBranch(String projectId, RefJson branch) {
         try {
             branchIndex.update(branch);
             branchRepository.save(b);
-            Set<String> docIds = new HashSet<>();
-            for (Node n: nodeRepository.findAllByDeleted(false)) {
-                docIds.add(n.getDocId());
+            if(!fromCommit) {
+                Set<String> docIds = new HashSet<>();
+                for (Node n: nodeRepository.findAllByDeleted(false)) {
+                    docIds.add(n.getDocId());
+                }
+
+                try { nodeIndex.addToRef(docIds); } catch(Exception e) {}
             }
-            try { nodeIndex.addToRef(docIds); } catch(Exception e) {}
             eventPublisher.forEach((pub) -> pub.publish(
                 EventObject.create(projectId, branch.getId(), "branch_created", branch)));
             return branch;
@@ -181,6 +199,60 @@ public RefJson createBranch(String projectId, RefJson branch) {
             throw new InternalErrorException(e);
         }
     }
+    public RefJson createBranchfromCommit(String projectId, RefJson parentCommitIdRef, NodeService nodeService) {
+        Instant now = Instant.now();
+
+        if(parentCommitIdRef.getParentCommitId().isEmpty()){
+            throw new BadRequestException(new RefsResponse().addMessage("parentCommitId not provided " + now.toString()));
+        }
+
+        ContextHolder.setContext(projectId);
+        Optional<Commit> parentCommit = commitRepository.findByCommitId(parentCommitIdRef.getParentCommitId());
+
+        // Get Commit object
+        String parentCommitID = parentCommit.map(Commit::getCommitId).orElseThrow(() ->
+            new BadRequestException(new RefsResponse().addMessage("parentCommitId not found " + now.toString())));
+
+        // Get Commit parentRef, the branch will be created from this
+        String parentRef = parentCommit.map(Commit::getBranchId).orElseThrow(() ->
+            new BadRequestException(new RefsResponse().addMessage("Ref from parentCommitId not found"  + now.toString())));
+
+        parentCommitIdRef.setParentRefId(parentRef);
+        ContextHolder.setContext(projectId, parentRef);
+
+        RefJson branchFromCommit = this.createBranch(projectId, parentCommitIdRef);
+        ContextHolder.setContext(projectId, branchFromCommit.getId());
+
+        // Get current nodes from database
+        List<Node> nodes = nodeRepository.findAll();
+        // Get elements from index
+        Collection<ElementJson> result = nodeGetHelper.processGetJsonFromNodes(nodes, parentCommitID, nodeService)
+            .getActiveElementMap().values();
+
+        Map<String, ElementJson> nodeCommitData = new HashMap<>();
+        for (ElementJson element : result) {
+            nodeCommitData.put(element.getId(), element);
+        }
+
+        // Update database table to match index
+        Set<String> docIds = new HashSet<>();
+        for (Node node : nodes) {
+            if(nodeCommitData.containsKey(node.getNodeId())){
+                node.setDocId(nodeCommitData.get(node.getNodeId()).getDocId());
+                node.setLastCommit(nodeCommitData.get(node.getNodeId()).getCommitId());
+                node.setDeleted(false);
+                docIds.add(node.getDocId());
+            } else {
+                node.setDeleted(true);
+            }
+        }
+        nodeRepository.updateAll(nodes);
+        branchFromCommit.setStatus("created");
+        branchIndex.update(branchFromCommit);
+        try { nodeIndex.addToRef(docIds); } catch(Exception e) {}
+
+        return branchFromCommit;
+    }
 
     public RefsResponse deleteBranch(String projectId, String id) {
         ContextHolder.setContext(projectId);

crud/src/main/java/org/openmbee/mms/crud/services/NodeGetHelper.java

@@ -13,14 +13,24 @@
 import org.openmbee.mms.data.domains.scoped.Branch;
 import org.openmbee.mms.data.domains.scoped.Commit;
 import org.openmbee.mms.data.domains.scoped.Node;
+import org.openmbee.mms.core.dao.CommitIndexDAO;
+import org.openmbee.mms.json.CommitJson;
 import org.openmbee.mms.json.ElementJson;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import static org.openmbee.mms.core.config.ContextHolder.getContext;
 
 @Service
 public class NodeGetHelper extends NodeOperation {
 
+    protected CommitIndexDAO commitIndex;
+
+    @Autowired
+    public void setCommitIndex(CommitIndexDAO commitIndex) {
+        this.commitIndex = commitIndex;
+    }
+
     public NodeGetInfo processGetJsonFromNodes(List<Node> nodes, NodeService service) {
         NodeGetInfo info = initInfoFromNodes(nodes, null);
         return processLatest(info, service);
@@ -122,21 +132,45 @@ private NodeGetInfo processCommit(NodeGetInfo info, String commitId, NodeService
                 Optional<ElementJson> e = nodeIndex.getElementLessThanOrEqualTimestamp(nodeId,
                     formatter.format(time), refCommitIds);
                 if (e.isPresent()) { // found version of element at commit time
-                    //TODO determine if element was deleted at the time?
-                    e.get().setRefId(ContextHolder.getContext().getBranchId());
-                    info.getActiveElementMap().put(nodeId, e.get());
+                    Instant realModified = Instant.from(formatter.parse(e.get().getModified()));
+                    if (elementDeleted(nodeId, commitId, time, realModified, refCommitIds)) {
+                        rejectDeleted(info, nodeId, e.get());
+                    } else {
+                        e.get().setRefId(ContextHolder.getContext().getBranchId());
+                        info.getActiveElementMap().put(nodeId, e.get());
+                    }
                 } else {
                     rejectNotFound(info, nodeId); // element not found at commit time
                 }
             } else if (info.getExistingNodeMap().get(nodeId).isDeleted()) { // latest element is before commit, but deleted
-                rejectDeleted(info, nodeId, indexElement);
+                if (refCommitIds == null) { // need list of commitIds of current ref to filter on
+                    refCommitIds = getRefCommitIds(time);
+                }
+                if (elementDeleted(nodeId, commitId, time, modified, refCommitIds)) {
+                    rejectDeleted(info, nodeId, indexElement);
+                } else {
+                    info.getActiveElementMap().put(nodeId, indexElement);
+                }
             } else { // latest element version is version at commit, not deleted
                 info.getActiveElementMap().put(nodeId, indexElement);
             }
         }
         return info;
     }
 
+    private boolean elementDeleted(String nodeId, String commitId, Instant time, Instant modified, List<String> refCommitIds) {
+        List<CommitJson> commits = commitIndex.elementDeletedHistory(nodeId, refCommitIds);
+        for (CommitJson c: commits) {
+            Instant deletedTime = Instant.from(formatter.parse(c.getCreated()));
+            if ((deletedTime.isBefore(time) || c.getId().equals(commitId)) && deletedTime.isAfter(modified)) {
+                //there's a delete between element last modified time and requested commit time
+                //or element is deleted at commit
+                return true;
+            }
+        }
+        return false;
+    }
+
     public NodeGetInfo processGetJson(List<ElementJson> elements, Instant time, NodeService service) {
         Optional<Branch> ref = branchRepository.findByBranchId(getContext().getBranchId());
         if (ref.isPresent()) {

data/data.gradle

@@ -10,4 +10,4 @@ dependencies {
     api commonDependencies.'hibernate-core'
 
     testImplementation commonDependencies.'spring-boot-starter-test'
-}
+}

docs/conf.py

@@ -26,7 +26,7 @@
 master_doc = 'index'
 
 # The full version, including alpha/beta/rc tags
-release = '4.0.19'
+release = '4.0.20'
 
 
 # -- General configuration ---------------------------------------------------
@@ -57,4 +57,4 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ['_static']

docs/requirements.txt

@@ -0,0 +1 @@
+sphinx_rtd_theme

elastic/README.rst

@@ -21,6 +21,16 @@ The following are a list of options to configure the Elastic Module for MMS.
   elasticsearch.http
     The transport protocol to use to connect to the Elasticsearch server or cluster. Required.
 
+  elasticsearch.username
+    Username, Optional.
+
+    | `Default: null`
+
+  elasticsearch.password
+    Password, Optional.
+
+    | `Default: null`
+
   elasticsearch.limit.result
     The maximum number of results a single search request should return. Optional.
 

elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java

@@ -124,6 +124,33 @@ private QueryBuilder getCommitHistoryQuery(String id, Set<String> commitIds) {
         return query;
     }
 
+    @Override
+    public List<CommitJson> elementDeletedHistory(String id, Collection<String> commitIds) {
+        QueryBuilder query = QueryBuilders.boolQuery()
+            .filter(QueryBuilders.termQuery("deleted.id", id))
+            .filter(QueryBuilders.termsQuery(CommitJson.ID, commitIds));
+        try {
+            List<CommitJson> commits = new ArrayList<>();
+            SearchHits hits = getCommitResults(query);
+            if (hits.getTotalHits().value == 0) {
+                return new ArrayList<>();
+            }
+            for (SearchHit hit : hits.getHits()) {
+                Map<String, Object> source = hit.getSourceAsMap();// gets "_source"
+                CommitJson ob = newInstance();
+                ob.putAll(source);
+                ob.remove(CommitJson.ADDED);
+                ob.remove(CommitJson.UPDATED);
+                ob.remove(CommitJson.DELETED);
+                commits.add(ob);
+            }
+            return commits;
+        } catch (IOException e) {
+            logger.error(e.getMessage(), e);
+            throw new InternalErrorException(e);
+        }
+    }
+
     /**
      * Returns the commit history of a element
      * <p> Returns a list of commit metadata for the specified id