From 5b2163ffae7bf4013e0c3799fb3c942f6e310a43 Mon Sep 17 00:00:00 2001 From: Erick Tryzelaar Date: Fri, 14 Feb 2014 17:05:23 -0800 Subject: [PATCH 1/5] Initial support for dendrite branching to be fed into git --- pom.xml | 7 ++ .../dendrite/services/HistoryService.java | 33 ++++++-- .../web/controller/BranchController.java | 79 ++++++++++++++++--- .../web/controller/GraphExportController.java | 69 +++++++++++----- 4 files changed, 151 insertions(+), 37 deletions(-) diff --git a/pom.xml b/pom.xml index 52e5cdc..6195560 100644 --- a/pom.xml +++ b/pom.xml @@ -468,6 +468,13 @@ blueprints-graph-jung 2.4.0 + + + + org.eclipse.jgit + org.eclipse.jgit + 3.2.0.201312181205-r + diff --git a/src/main/java/org/lab41/dendrite/services/HistoryService.java b/src/main/java/org/lab41/dendrite/services/HistoryService.java index 4ce8128..f1fe8e9 100644 --- a/src/main/java/org/lab41/dendrite/services/HistoryService.java +++ b/src/main/java/org/lab41/dendrite/services/HistoryService.java @@ -3,6 +3,10 @@ import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.lab41.dendrite.metagraph.DendriteGraph; +import org.lab41.dendrite.metagraph.models.ProjectMetadata; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -18,7 +22,9 @@ @Service public class HistoryService { - Logger logger = LoggerFactory.getLogger(HistoryService.class); + private static Logger logger = LoggerFactory.getLogger(HistoryService.class); + + private String historyStorage; @Autowired(required = true) public HistoryService(@Value("${history.properties}") String pathToProperties, ResourceLoader resourceLoader) throws IOException, ConfigurationException { @@ -31,13 +37,26 @@ public HistoryService(@Value("${history.properties}") String pathToProperties, R // create directory historyStorage = configuration.getString("history.storage"); - File file = new File(historyStorage); - file.mkdirs(); } - public String getHistoryStorage() { - return historyStorage; - } + public Git projectGitRepository(ProjectMetadata projectMetadata) throws GitAPIException, IOException { + File gitDir = new File(historyStorage, projectMetadata.getId()); - private String historyStorage; + // Make the target directory. + Git git; + if (gitDir.exists()) { + git = Git.open(gitDir); + } else { + logger.debug("Creating git repository: %s", gitDir); + + gitDir.mkdirs(); + + git = Git.init() + .setDirectory(gitDir) + .setBare(false) + .call(); + } + + return git; + } } diff --git a/src/main/java/org/lab41/dendrite/web/controller/BranchController.java b/src/main/java/org/lab41/dendrite/web/controller/BranchController.java index dc49d17..a501dfc 100644 --- a/src/main/java/org/lab41/dendrite/web/controller/BranchController.java +++ b/src/main/java/org/lab41/dendrite/web/controller/BranchController.java @@ -1,15 +1,18 @@ package org.lab41.dendrite.web.controller; -import org.codehaus.jettison.json.JSONObject; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.api.errors.NoMessageException; +import org.eclipse.jgit.api.errors.WrongRepositoryStateException; import org.lab41.dendrite.jobs.BranchCommitJob; import org.lab41.dendrite.jobs.BranchCommitSubsetJob; +import org.lab41.dendrite.metagraph.DendriteGraph; import org.lab41.dendrite.metagraph.models.BranchMetadata; import org.lab41.dendrite.metagraph.models.GraphMetadata; import org.lab41.dendrite.metagraph.models.JobMetadata; import org.lab41.dendrite.metagraph.models.ProjectMetadata; import org.lab41.dendrite.metagraph.MetaGraphTx; +import org.lab41.dendrite.services.HistoryService; import org.lab41.dendrite.services.MetaGraphService; import org.lab41.dendrite.web.beans.CreateBranchBean; import org.lab41.dendrite.web.beans.CreateBranchSubsetNStepsBean; @@ -27,6 +30,7 @@ import org.springframework.web.bind.annotation.RequestMethod; import javax.validation.Valid; +import java.io.IOException; import java.text.SimpleDateFormat; import java.util.*; @@ -40,6 +44,9 @@ public class BranchController { @Autowired TaskExecutor taskExecutor; + @Autowired + HistoryService historyService; + @RequestMapping(value = "/branches", method = RequestMethod.GET) public ResponseEntity> getBranches() { @@ -87,6 +94,7 @@ public ResponseEntity> deleteBranch(@PathVariable String bra Map response = new HashMap<>(); MetaGraphTx tx = metaGraphService.newTransaction(); + BranchMetadata branchMetadata = tx.getBranch(branchId); if (branchMetadata == null) { @@ -96,8 +104,21 @@ public ResponseEntity> deleteBranch(@PathVariable String bra return new ResponseEntity<>(HttpStatus.NOT_FOUND); } + String branchName = branchMetadata.getName(); + try { tx.deleteBranch(branchMetadata); + + Git git = historyService.projectGitRepository(branchMetadata.getProject()); + try { + git.branchDelete() + .setBranchNames(branchName) + .call(); + } finally { + git.close(); + } + + tx.commit(); } catch (Exception e) { response.put("status", "error"); response.put("msg", e.toString()); @@ -105,10 +126,16 @@ public ResponseEntity> deleteBranch(@PathVariable String bra return new ResponseEntity<>(HttpStatus.BAD_REQUEST); } - response.put("msg", "deleted"); + try { - // Commit must come after all branch access. - tx.commit(); + response.put("msg", "deleted"); + + // Commit must come after all branch access. + tx.commit(); + } catch (Throwable t) { + tx.rollback(); + throw t; + } return new ResponseEntity<>(response, HttpStatus.OK); } @@ -176,7 +203,7 @@ public ResponseEntity> getBranch(@PathVariable String projec public ResponseEntity> createBranch(@PathVariable String projectId, @PathVariable String branchName, @Valid @RequestBody CreateBranchBean item, - BindingResult result) { + BindingResult result) throws GitAPIException, IOException { Map response = new HashMap<>(); @@ -216,6 +243,15 @@ public ResponseEntity> createBranch(@PathVariable String pro JobMetadata jobMetadata = tx.createJob(projectMetadata); + Git git = historyService.projectGitRepository(projectMetadata); + try { + git.branchCreate() + .setName(branchName) + .call(); + } finally { + git.close(); + } + // Commit must come after all branch access. tx.commit(); @@ -269,7 +305,7 @@ public ResponseEntity> getCurrentBranch(@PathVariable String @RequestMapping(value = "/projects/{projectId}/current-branch", method = RequestMethod.PUT) public ResponseEntity> getCurrentBranch(@PathVariable String projectId, @Valid @RequestBody UpdateCurrentBranchBean item, - BindingResult result) { + BindingResult result) throws GitAPIException, IOException { Map response = new HashMap<>(); @@ -300,6 +336,15 @@ public ResponseEntity> getCurrentBranch(@PathVariable String projectMetadata.setCurrentBranch(branchMetadata); + Git git = historyService.projectGitRepository(projectMetadata); + try { + git.checkout() + .setName(branchName) + .call(); + } finally { + git.close(); + } + response.put("msg", "current branch changed"); // Commit must come after all branch access. @@ -309,7 +354,7 @@ public ResponseEntity> getCurrentBranch(@PathVariable String } @RequestMapping(value = "/projects/{projectId}/current-branch/commit", method = RequestMethod.POST) - public ResponseEntity> commitBranch(@PathVariable String projectId) { + public ResponseEntity> commitBranch(@PathVariable String projectId) throws GitAPIException, IOException { Map response = new HashMap<>(); MetaGraphTx tx = metaGraphService.newTransaction(); @@ -332,6 +377,22 @@ public ResponseEntity> commitBranch(@PathVariable String pro JobMetadata jobMetadata = tx.createJob(projectMetadata); + try { + Git git = historyService.projectGitRepository(projectMetadata); + try { + git.commit() + .setAuthor("user", "user@example.com") + .setMessage("commit") + .call(); + } finally { + git.close(); + } + + } catch (Throwable t) { + tx.rollback(); + throw t; + } + tx.commit(); // We can't pass the values directly because they'll live in a separate thread. diff --git a/src/main/java/org/lab41/dendrite/web/controller/GraphExportController.java b/src/main/java/org/lab41/dendrite/web/controller/GraphExportController.java index 078e00b..da257ba 100644 --- a/src/main/java/org/lab41/dendrite/web/controller/GraphExportController.java +++ b/src/main/java/org/lab41/dendrite/web/controller/GraphExportController.java @@ -20,8 +20,12 @@ import com.tinkerpop.blueprints.util.io.graphml.GraphMLWriter; import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; import org.lab41.dendrite.metagraph.DendriteGraph; import org.lab41.dendrite.metagraph.DendriteGraphTx; +import org.lab41.dendrite.metagraph.MetaGraphTx; +import org.lab41.dendrite.metagraph.models.GraphMetadata; import org.lab41.dendrite.services.HistoryService; import org.lab41.dendrite.services.MetaGraphService; import org.lab41.dendrite.web.beans.GraphExportBean; @@ -113,7 +117,7 @@ public ResponseEntity export(@PathVariable String graphId, @RequestMapping(value = "/api/graphs/{graphId}/file-save", method = RequestMethod.POST) public ResponseEntity> save(@PathVariable String graphId, @Valid GraphExportBean item, - BindingResult result) { + BindingResult result) throws IOException, GitAPIException { Map response = new HashMap<>(); @@ -123,6 +127,19 @@ public ResponseEntity> save(@PathVariable String graphId, return new ResponseEntity<>(response, HttpStatus.BAD_REQUEST); } + MetaGraphTx metaGraphTx = metaGraphService.buildTransaction().readOnly().start(); + GraphMetadata graphMetadata; + Git git; + + try { + graphMetadata = metaGraphTx.getGraph(graphId); + git = historyService.projectGitRepository(graphMetadata.getProject()); + metaGraphTx.commit(); + } catch (Throwable t) { + metaGraphTx.rollback(); + throw t; + } + DendriteGraph graph = metaGraphService.getGraph(graphId); if (graph == null) { response.put("status", "error"); @@ -134,30 +151,40 @@ public ResponseEntity> save(@PathVariable String graphId, // extract the storage location for the history String format = item.getFormat(); - String projectId = item.getProjectId(); - String historyStorageLocation = historyService.getHistoryStorage() + "/" + projectId; - - // Make the target directory. - new File(historyStorageLocation).mkdirs(); DendriteGraphTx tx = graph.buildTransaction().readOnly().start(); try { - if (format.equalsIgnoreCase("GraphSON")) { - String path = new File(historyStorageLocation, graphId + ".json").getPath(); - GraphSONWriter.outputGraph(tx, path); - } else if (format.equalsIgnoreCase("GraphML")) { - String path = new File(historyStorageLocation, graphId + ".xml").getPath(); - GraphMLWriter.outputGraph(tx, path); - } else if (format.equalsIgnoreCase("GML")) { - String path = new File(historyStorageLocation, graphId + ".gml").getPath(); - GMLWriter.outputGraph(tx, path); - } else { - tx.rollback(); - - response.put("status", "error"); - response.put("msg", "unknown format '" + format + "'"); - return new ResponseEntity<>(response, HttpStatus.BAD_REQUEST); + try { + String path; + + if (format.equalsIgnoreCase("GraphSON")) { + path = new File(git.getRepository().getDirectory(), graphId + ".json").getPath(); + GraphSONWriter.outputGraph(tx, path); + } else if (format.equalsIgnoreCase("GraphML")) { + path = new File(git.getRepository().getDirectory(), graphId + ".xml").getPath(); + GraphMLWriter.outputGraph(tx, path); + } else if (format.equalsIgnoreCase("GML")) { + path = new File(git.getRepository().getDirectory(), graphId + ".gml").getPath(); + GMLWriter.outputGraph(tx, path); + } else { + tx.rollback(); + + response.put("status", "error"); + response.put("msg", "unknown format '" + format + "'"); + return new ResponseEntity<>(response, HttpStatus.BAD_REQUEST); + } + + git.add() + .addFilepattern(".") + .call(); + + git.commit() + .setAuthor("user", "user@example.com") + .setMessage("commit") + .call(); + } finally { + git.close(); } } catch (IOException e) { tx.rollback(); From 921b7ad8cae93d938008f4e0989da4a879ca40f0 Mon Sep 17 00:00:00 2001 From: Erick Tryzelaar Date: Fri, 14 Feb 2014 17:05:56 -0800 Subject: [PATCH 2/5] Disable committing an arbitrary branch Not sure how to get this to work with ungit at the moment. --- .../org/lab41/dendrite/web/controller/BranchController.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/lab41/dendrite/web/controller/BranchController.java b/src/main/java/org/lab41/dendrite/web/controller/BranchController.java index a501dfc..bc2eeaa 100644 --- a/src/main/java/org/lab41/dendrite/web/controller/BranchController.java +++ b/src/main/java/org/lab41/dendrite/web/controller/BranchController.java @@ -411,6 +411,7 @@ public ResponseEntity> commitBranch(@PathVariable String pro return new ResponseEntity<>(response, HttpStatus.OK); } + /* @RequestMapping(value = "/projects/{projectId}/branches/{branchName}/commit", method = RequestMethod.POST) public ResponseEntity> commitBranch(@PathVariable String projectId, @PathVariable String branchName) { @@ -453,6 +454,7 @@ public ResponseEntity> commitBranch(@PathVariable String pro return new ResponseEntity<>(response, HttpStatus.OK); } + */ @RequestMapping(value = "/projects/{projectId}/current-branch/commit-subset", method = RequestMethod.POST) public ResponseEntity> commitSubsetBranch(@PathVariable String projectId, From 78c874bd5521c0c7511c37a132f7c01c23ff2cba Mon Sep 17 00:00:00 2001 From: Erick Tryzelaar Date: Fri, 14 Feb 2014 17:41:32 -0800 Subject: [PATCH 3/5] Make sure .git exists in the project history dir --- .../java/org/lab41/dendrite/services/HistoryService.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/lab41/dendrite/services/HistoryService.java b/src/main/java/org/lab41/dendrite/services/HistoryService.java index f1fe8e9..d2cb37d 100644 --- a/src/main/java/org/lab41/dendrite/services/HistoryService.java +++ b/src/main/java/org/lab41/dendrite/services/HistoryService.java @@ -37,6 +37,9 @@ public HistoryService(@Value("${history.properties}") String pathToProperties, R // create directory historyStorage = configuration.getString("history.storage"); + + // Make sure the directory exists. + new File(historyStorage).mkdirs(); } public Git projectGitRepository(ProjectMetadata projectMetadata) throws GitAPIException, IOException { @@ -44,7 +47,7 @@ public Git projectGitRepository(ProjectMetadata projectMetadata) throws GitAPIEx // Make the target directory. Git git; - if (gitDir.exists()) { + if (gitDir.exists() && new File(gitDir, ".git").exists()) { git = Git.open(gitDir); } else { logger.debug("Creating git repository: %s", gitDir); From 2d2254e6c179ec4a40a7ac48d00aa3a44e0663df Mon Sep 17 00:00:00 2001 From: Erick Tryzelaar Date: Wed, 26 Mar 2014 13:41:13 -0700 Subject: [PATCH 4/5] Fix deploying dendrite to tomcat --- pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pom.xml b/pom.xml index 6195560..77c158d 100644 --- a/pom.xml +++ b/pom.xml @@ -592,6 +592,9 @@ tomcat7-maven-plugin 2.2 + ${tomcat.url} + ${tomcat.server} + /dendrite ${HADOOP_CONF_DIR} From 7ff88c600978cd3af8b12864d141f5a2c2da6234 Mon Sep 17 00:00:00 2001 From: Erick Tryzelaar Date: Wed, 26 Mar 2014 13:41:51 -0700 Subject: [PATCH 5/5] git commits should use the authenticated user's username --- .../org/lab41/dendrite/web/controller/BranchController.java | 6 +++++- .../dendrite/web/controller/GraphExportController.java | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/lab41/dendrite/web/controller/BranchController.java b/src/main/java/org/lab41/dendrite/web/controller/BranchController.java index bc2eeaa..915c664 100644 --- a/src/main/java/org/lab41/dendrite/web/controller/BranchController.java +++ b/src/main/java/org/lab41/dendrite/web/controller/BranchController.java @@ -22,6 +22,8 @@ import org.springframework.core.task.TaskExecutor; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Controller; import org.springframework.validation.BindingResult; import org.springframework.web.bind.annotation.PathVariable; @@ -355,6 +357,8 @@ public ResponseEntity> getCurrentBranch(@PathVariable String @RequestMapping(value = "/projects/{projectId}/current-branch/commit", method = RequestMethod.POST) public ResponseEntity> commitBranch(@PathVariable String projectId) throws GitAPIException, IOException { + Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); + Map response = new HashMap<>(); MetaGraphTx tx = metaGraphService.newTransaction(); @@ -381,7 +385,7 @@ public ResponseEntity> commitBranch(@PathVariable String pro Git git = historyService.projectGitRepository(projectMetadata); try { git.commit() - .setAuthor("user", "user@example.com") + .setAuthor(authentication.getName(), "") .setMessage("commit") .call(); } finally { diff --git a/src/main/java/org/lab41/dendrite/web/controller/GraphExportController.java b/src/main/java/org/lab41/dendrite/web/controller/GraphExportController.java index da257ba..7250103 100644 --- a/src/main/java/org/lab41/dendrite/web/controller/GraphExportController.java +++ b/src/main/java/org/lab41/dendrite/web/controller/GraphExportController.java @@ -36,6 +36,8 @@ import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Controller; import org.springframework.validation.BindingResult; import org.springframework.web.bind.annotation.PathVariable; @@ -118,6 +120,7 @@ public ResponseEntity export(@PathVariable String graphId, public ResponseEntity> save(@PathVariable String graphId, @Valid GraphExportBean item, BindingResult result) throws IOException, GitAPIException { + Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); Map response = new HashMap<>(); @@ -180,7 +183,7 @@ public ResponseEntity> save(@PathVariable String graphId, .call(); git.commit() - .setAuthor("user", "user@example.com") + .setAuthor(authentication.getName(), "") .setMessage("commit") .call(); } finally {