Merge pull request #260 from Open-MBEE/fix/persistence_refactor_fixes

Fix/persistence refactor fixes

.circleci/config.yml

@@ -20,27 +20,36 @@ jobs:
 
       - setup_remote_docker
 
-      - run: 
+      - run:
           name: "Create and start all services from the docker-compose configuration"
           command: |
             cp example/src/main/resources/application.properties.example ./example/src/main/resources/application.properties
+            mkdir -p jacocoJars
+            curl -L -o ./jacocoJars/jacocoagent.jar "https://repo1.maven.org/maven2/org/jacoco/org.jacoco.agent/0.8.7/org.jacoco.agent-0.8.7-runtime.jar"
+            curl -L -o ./jacocoJars/jacococli.jar "https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/0.8.7/org.jacoco.cli-0.8.7-nodeps.jar"
             docker-compose up --build -d
             docker run --network container:mms curlimages/curl --retry 8 --retry-delay 10 --retry-max-time 90 --retry-connrefused http://mms:8080/healthcheck
 
-      - run: 
+      - run:
           name: "Run and test Postman Collection"
           command: |
             docker create -v /etc/newman --name mms_test_configs alpine:3.4 /bin/true
             docker cp example/. mms_test_configs:/etc/newman
-            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run crud.postman_collection.json -e test-env.json --delay-request 1000
-            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run cameo.postman_collection.json -e test-env.json --delay-request 1000
+            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run crud.postman_collection.json -e test-env.json --delay-request 500
+            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run getAtCommits.postman_collection.json -e test-env.json --delay-request 500
+            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run makeBranchFromCommit.postman_collection.json -e test-env.json --delay-request 500
+            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run cameo.postman_collection.json -e test-env.json --delay-request 500
             docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run jupyter.postman_collection.json -e test-env.json --delay-request 500
             docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run localauth.postman_collection.json -e test-env.json --delay-request 500
-            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run permissions.postman_collection.json -e test-env.json --delay-request 1000
+            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run permissions.postman_collection.json -e test-env.json --delay-request 500
             docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run search.postman_collection.json -e test-env.json --delay-request 1000
-            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run artifacts.postman_collection.json -e test-env.json --delay-request 1000
+            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run artifacts.postman_collection.json -e test-env.json --delay-request 500
             docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run elastic.postman_collection.json -e test-env.json --delay-request 500
             docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run groups.postman_collection.json -e test-env.json --delay-request 500
+            
+            docker stop mms
+            mkdir -p jacocoOutput
+            docker cp mms:/mms/jacocoOutput/jacoco-it.exec ./jacocoOutput/jacoco-it.exec
 
       - persist_to_workspace:
             root: /home/circleci/
@@ -51,10 +60,17 @@ jobs:
       executor: openjdk_executor
       working_directory: /home/circleci/mms
       steps:
-          - checkout
+          - attach_workspace:
+                at: ~/
           - run:
                 name: Build classes to scan
-                command: ./gradlew build -x test
+                command: |
+                    ./gradlew build -x test
+                    mkdir -p TEMPORARYclassfiles
+                    cp -r */build/classes/java/main/* TEMPORARYclassfiles/
+                    java -jar jacocoJars/jacococli.jar report jacocoOutput/jacoco-it.exec --classfiles TEMPORARYclassfiles/ --xml ./jacocoOutput/jacocoXMLReport
+                    rm -r TEMPORARYclassfiles
+
           - sonarcloud/scan
 
   deploy_snapshot:
@@ -85,6 +101,8 @@ workflows:
                   tags:
                       only: /[0-9.]+(-(a|b|rc)[0-9]+)?/
         - scan:
+              requires:
+                  - build_and_test
               filters:
                   tags:
                       only: /[0-9.]+(-(a|b|rc)[0-9]+)?/

.readthedocs.yaml

@@ -17,6 +17,6 @@ sphinx:
 
 # We recommend specifying your dependencies to enable reproducible builds:
 # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
-# python:
-#   install:
-#   - requirements: docs/requirements.txt
+python:
+  install:
+    - requirements: docs/requirements.txt

Dockerfile

@@ -5,5 +5,5 @@ RUN ./gradlew --no-daemon bootJar --warning-mode all
 
 RUN find . -type f -name example-*.jar -not -iname '*javadoc*' -not -iname '*sources*' -exec cp '{}' '/app.jar' ';'
 ENV JDK_JAVA_OPTIONS "-XX:MaxRAMPercentage=90.0"
-ENTRYPOINT ["java", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "-jar", "/app.jar"]
+ENTRYPOINT ["java", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "-javaagent:jacocoJars/jacocoagent.jar=destfile=jacocoOutput/jacoco-it.exec,output=file", "-jar", "/app.jar"]
 EXPOSE 8080

build.gradle

@@ -65,6 +65,7 @@ subprojects {
 
     group = "org.openmbee.mms"
     version = rootProject.version
+    ext['jackson-bom.version'] = jacksonVersion
 
     Map<String, String> commonDependencies = rootProject.ext.commonDependencies
 
@@ -160,4 +161,4 @@ subprojects {
             sign publishing.publications.mavenJava
         }
     }
-}
+}

cameo/src/main/java/org/openmbee/mms/cameo/services/CameoCommitService.java

@@ -11,7 +11,7 @@
 public class CameoCommitService extends DefaultCommitService implements CommitService {
     @Override
     public boolean isProjectNew(String projectId) {
-        List<CommitJson> commits = commitPersistence.findAllByProjectId(projectId);
+        List<CommitJson> commits = commitPersistence.findByProjectAndRefAndTimestampAndLimit(projectId, "master", null, 2);
         return commits == null || commits.size() <= 1; // a cameo project gets 1 auto commit, so its still "new"
     }
 }

cameo/src/main/java/org/openmbee/mms/cameo/services/CameoNodeService.java

@@ -10,6 +10,7 @@
 import org.openmbee.mms.core.services.HierarchicalNodeService;
 import org.openmbee.mms.core.services.NodeChangeInfo;
 import org.openmbee.mms.core.services.NodeGetInfo;
+import org.openmbee.mms.crud.CrudConstants;
 import org.openmbee.mms.crud.services.DefaultNodeService;
 import org.openmbee.mms.json.CommitJson;
 import org.openmbee.mms.json.ElementJson;
@@ -42,17 +43,17 @@ public void setMss(MethodSecurityService mss) {
     public ElementsResponse read(String projectId, String refId, ElementsRequest req,
             Map<String, String> params) {
 
-        String commitId = params.getOrDefault(CameoConstants.COMMITID, null);
+        String commitId = params.getOrDefault(CrudConstants.COMMITID, null);
         if (commitId == null) {
             Optional<CommitJson> commitJson = commitPersistence.findLatestByProjectAndRef(projectId, refId);
             if (!commitJson.isPresent()) {
                 throw new InternalErrorException("Could not find latest commit for project and ref");
             }
-            commitId = commitJson.get().getId();   
+            commitId = commitJson.get().getId();
         }
 
         NodeGetInfo info = getNodePersistence().findAll(projectId, refId, commitId, req.getElements());
-
+        info.getActiveElementMap().values().forEach((e) -> e.setRefId(refId));
         if (!info.getRejected().isEmpty()) {
             //continue looking in visible mounted projects for elements if not all found
             NodeGetInfo curInfo = info;
@@ -61,9 +62,11 @@ public ElementsResponse read(String projectId, String refId, ElementsRequest req
 
             int i = 1; //0 is entry project, already gotten
             while (!curInfo.getRejected().isEmpty() && i < usages.size()) {
+                final int j = i;
                 ElementsRequest reqNext = buildRequest(curInfo.getRejected().keySet());
                 //TODO use the right commitId in child if commitId is present in params :: same commit Id is not working for child
                 curInfo = getNodePersistence().findAll(usages.get(i).getFirst(), usages.get(i).getSecond(), "", reqNext.getElements());
+                curInfo.getActiveElementMap().values().forEach((e) -> e.setRefId(usages.get(j).getSecond()));
                 info.getActiveElementMap().putAll(curInfo.getActiveElementMap());
                 curInfo.getActiveElementMap().forEach((id, json) -> info.getRejected().remove(id));
                 curInfo.getRejected().forEach((id, rejection) -> {
@@ -78,6 +81,7 @@ public ElementsResponse read(String projectId, String refId, ElementsRequest req
         ElementsResponse response = new ElementsResponse();
         response.getElements().addAll(info.getActiveElementMap().values());
         response.setRejected(new ArrayList<>(info.getRejected().values()));
+        response.setCommitId(commitId);
         return response;
     }
 

cameo/src/main/java/org/openmbee/mms/cameo/services/CameoViewService.java

@@ -19,6 +19,7 @@
 import org.openmbee.mms.core.objects.ElementsResponse;
 import org.openmbee.mms.core.services.NodeChangeInfo;
 import org.openmbee.mms.core.services.NodeGetInfo;
+import org.openmbee.mms.crud.domain.JsonDomain;
 import org.openmbee.mms.json.ElementJson;
 import org.openmbee.mms.view.services.PropertyData;
 import org.openmbee.mms.view.services.ViewService;
@@ -37,7 +38,7 @@ public ElementsResponse getDocuments(String projectId, String refId, Map<String,
             Optional<ElementJson> parent = getFirstRelationshipOfType(projectId, refId, commitId, e,
                 List.of(CameoNodeType.GROUP.getValue()), CameoConstants.OWNERID);
             parent.ifPresent(elementJson -> e.put(CameoConstants.SITECHARACTERIZATIONID, elementJson.getId()));
-            
+
         }
         return res;
     }
@@ -64,7 +65,7 @@ public void addChildViews(ElementsResponse res, Map<String, String> params) {
                 }
                 ElementsResponse ownedAttributes = this.read(element.getProjectId(), element.getRefId(),
                     buildRequest(ownedAttributeIds), params);
-                List<ElementJson> filtered = filter(ownedAttributeIds, ownedAttributes.getElements());
+                List<ElementJson> filtered = JsonDomain.filter(ownedAttributeIds, ownedAttributes.getElements());
                 List<Map> childViews = new ArrayList<>();
                 for (ElementJson attr : filtered) {
                     String childId = (String) attr.get(CameoConstants.TYPEID);
@@ -86,8 +87,8 @@ public ElementsResponse getGroups(String projectId, String refId, Map<String, St
         List<ElementJson> groups = getNodePersistence().findAllByNodeType(projectId, refId, commitId,
             CameoNodeType.GROUP.getValue());
 
-        ElementsResponse res = this.read(projectId, refId, buildRequestFromJsons(groups), params);
-        for (ElementJson e: res.getElements()) {
+        ElementsResponse res = new ElementsResponse().setElements(groups);
+        for (ElementJson e: groups) {
             Optional<ElementJson> parent = getFirstRelationshipOfType(projectId, refId, commitId, e,
                 List.of(CameoNodeType.GROUP.getValue()), CameoConstants.OWNERID);
             parent.ifPresent(elementJson -> e.put(CameoConstants.PARENTID, elementJson.getId()));

core/src/main/java/org/openmbee/mms/core/builders/PermissionUpdatesResponseBuilder.java

@@ -5,8 +5,8 @@
 
 public class PermissionUpdatesResponseBuilder {
 
-    private Boolean inherit;
-    private Boolean isPublic;
+    protected Boolean inherit;
+    protected Boolean isPublic;
     private PermissionUpdateResponseBuilder usersBuilder = new PermissionUpdateResponseBuilder();
     private PermissionUpdateResponseBuilder groupsBuilder = new PermissionUpdateResponseBuilder();
 
@@ -56,13 +56,10 @@ public PermissionUpdateResponseBuilder getGroups() {
     }
 
     private Boolean or(Boolean a, Boolean b) {
-        if(a == b) {
-            return a;
-        }
-        if(a == null) {
+        if (a == null) {
             return b;
         }
-        if(b == null) {
+        if (b == null) {
             return a;
         }
         return a || b;

core/src/main/java/org/openmbee/mms/core/dao/CommitPersistence.java

@@ -23,7 +23,7 @@ public interface CommitPersistence {
 
     List<CommitJson> findByProjectAndRefAndTimestampAndLimit(String projectId, String refId, Instant timestamp, int limit);
 
-    List<CommitJson> elementHistory(String projectId, String elementId, Set<String> commitDocIds);
+    List<CommitJson> elementHistory(String projectId, String refId, String elementId);
 
     Optional<CommitJson> deleteById(String projectId, String commitId);
 

core/src/main/java/org/openmbee/mms/core/dao/NodePersistence.java

@@ -21,7 +21,7 @@ public interface NodePersistence {
 
     NodeChangeInfo commitChanges(NodeChangeInfo nodeChangeInfo);
 
-    NodeGetInfo findById(String elementId, String refId, String commitId, String id);
+    NodeGetInfo findById(String projectId, String refId, String commitId, String elementId);
 
     List<ElementJson> findAllByNodeType(String projectId, String refId, String commitId, int nodeType);
 

core/src/main/java/org/openmbee/mms/core/services/DefaultPermissionService.java

@@ -74,7 +74,7 @@ public void initProjectPerms(String projectId, boolean inherit, String creator)
 
     @Override
     public void initBranchPerms(String projectId, String branchId, boolean inherit, String creator) {
-        RefJson branch = getBranch(projectId, branchId, BRANCH_NOTFOUND_BEHAVIOR.CREATE);
+        RefJson branch = getBranch(projectId, branchId, BRANCH_NOTFOUND_BEHAVIOR.THROW);
         if(branch == null) {
             logger.error("Error initiating branch permissions " + projectId + " / " + branchId);
             throw new InternalErrorException("Could not initiate branch permissions");
@@ -166,7 +166,7 @@ public PermissionUpdateResponse updateBranchGroupPerms(PermissionUpdateRequest r
     @Override
     public PermissionUpdatesResponse setProjectInherit(boolean isInherit, String projectId) {
         PermissionUpdatesResponseBuilder responseBuilder = new PermissionUpdatesResponseBuilder();
-        responseBuilder.setInherit(true);
+        responseBuilder.setInherit(isInherit);
         ProjectJson project = getProject(projectId);
         PermissionsDelegate permissionsDelegate = permissionsDelegateUtil.getPermissionsDelegate(project);
         if (permissionsDelegate.setInherit(isInherit)) {
@@ -348,11 +348,13 @@ private RefJson getBranch(String projectId, String branchId, BRANCH_NOTFOUND_BEH
             switch (mode) {
                 case THROW:
                     throw new NotFoundException("Branch " + projectId + " " + branchId + " not found");
+                /* branch should never be created here
                 case CREATE:
                     RefJson b = new RefJson();
                     b.setProjectId(projectId);
                     b.setRefId(branchId);
                     return branchPersistence.save(b);
+                */
                 default:
                     //do nothing
                     break;

crud/src/main/java/org/openmbee/mms/crud/CrudConstants.java

@@ -12,4 +12,5 @@ public class CrudConstants {
     public static final String PROJECT = "Project";
     public static final String ORG = "Org";
     public static final String CREATED = "created";
+    public static final String CREATING = "creating";
 }

crud/src/main/java/org/openmbee/mms/crud/controllers/branches/BranchesController.java

@@ -74,7 +74,7 @@ public RefsResponse getRef(
 
     @PostMapping(consumes = MediaType.APPLICATION_JSON_VALUE)
     @PreAuthorize("@mss.hasProjectPrivilege(authentication, #projectId, 'PROJECT_CREATE_BRANCH', false)")
-    public RefsResponse createRefs(
+    public RefsResponse createRefs( //this can also handle update
         @PathVariable String projectId,
         @RequestBody RefsRequest projectsPost,
         Authentication auth) {
@@ -92,24 +92,24 @@ public RefsResponse createRefs(
                     response.addRejection(new Rejection(branch, 400, "Branch id is invalid."));
                     continue;
                 }
-
-                RefJson res;
-                branch.setCreator(auth.getName());
-                if (branch.getParentCommitId() == null || branch.getParentCommitId().isEmpty()) {
-                    Optional<RefJson> existingOptional = branchPersistence.findById(projectId, branch.getId());
-                    if (existingOptional.isPresent()) {
-                        //Branch exists, should merge the json
-                        branch.merge(existingOptional.get());
-                        res = branchService.updateBranch(projectId, branch);
-                    } else {
-                        res = branchService.createBranch(projectId, branch);
+                Optional<RefJson> existingOptional = branchPersistence.findById(projectId, branch.getId());
+                if (existingOptional.isPresent()) {
+                    //Branch exists, should merge the json, but cannot change parent ref or commit
+                    //if branch exists it will get resurrected if it was deleted
+                    RefJson existing = existingOptional.get();
+                    if (branch.getParentRefId() != null && !branch.getParentRefId().equals(existing.getParentRefId()) ||
+                        branch.getParentCommitId() != null && !branch.getParentCommitId().equals(existing.getParentCommitId())) {
+                        response.addRejection(new Rejection(branch, 400, "Cannot change existing branch's origin"));
+                        continue;
                     }
-                } else {
-                    //TODO implement branching from historical commit
-                    response.addRejection(new Rejection(branch, 400, "Branching from historical commits is not implemented."));
+                    branch.merge(existingOptional.get());
+                    RefJson res = branchService.updateBranch(projectId, branch);
+                    response.getRefs().add(res);
                     continue;
                 }
-
+                RefJson res;
+                branch.setCreator(auth.getName());
+                res = branchService.createBranch(projectId, branch);
                 permissionService.initBranchPerms(projectId, branch.getId(), true, auth.getName());
                 response.getRefs().add(res);
             } catch (MMSException e) {