Add pull request validation

.gitignore

@@ -0,0 +1,33 @@
+\#*\#
+*~
+*.bak
+*.swp
+*.swo
+.DS_Store
+
+atlassian-ide-plugin.xml
+*.class
+
+target/
+test-output/
+
+.project
+.classpath
+.settings/
+.metadata/
+
+.idea/
+*.iml
+
+prodDb.*
+
+*.log
+brooklyn*.log.*
+
+*brooklyn-persisted-state/
+
+*node_modules/
+*bower_components/
+
+ignored
+

pom.xml

@@ -0,0 +1,38 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.brooklyncentral.catalog</groupId>
+    <artifactId>parent</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <packaging>pom</packaging>
+
+    <name>Brooklyn-Central Community Catalog Directory</name>
+    <description>
+        Brooklyn-Central Community Catalog Directory parent / project root,
+        serving as the ancestor POM for all projects --
+        declaring versions, profiles, and the modules to build.
+    </description>
+
+    <url>http://catalog.cloudsoft.io/</url>
+
+    <properties>
+        <brooklyn.version>0.9.0</brooklyn.version>
+    </properties>
+
+    <modules>
+        <module>validation-testing</module>
+    </modules>
+
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>1.8</source>
+                    <target>1.8</target>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>

validation-testing/pom.xml

@@ -0,0 +1,84 @@
+<?xml version="1.0"?>
+<project
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+    xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.brooklyncentral.catalog</groupId>
+        <artifactId>parent</artifactId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>validation-testing</artifactId>
+
+    <name>Brooklyn-Central Community Catalog Blueprint Validation Testing</name>
+    <description>
+       Tests to determine if blueprint repositories are correctly formatted
+       and can be successfully scraped by the catalog server.
+    </description>
+
+    <properties>
+        <testng.version>6.1.1</testng.version>
+        <jgit.version>4.3.0.201604071810-r</jgit.version>
+        <guava.version>19.0</guava.version>
+        <commons-io.version>2.4</commons-io.version>
+        <slf4j-simple.version>1.7.21</slf4j-simple.version>
+        <util-validator.version>1.0-SNAPSHOT</util-validator.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.testng</groupId>
+            <artifactId>testng</artifactId>
+            <version>${testng.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.eclipse.jgit</groupId>
+            <artifactId>org.eclipse.jgit</artifactId>
+            <version>${jgit.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>${guava.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <version>${commons-io.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-simple</artifactId>
+            <version>${slf4j-simple.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.cloudsoft.catalog</groupId>
+            <artifactId>util-validator</artifactId>
+            <version>${util-validator.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <repositories>
+        <repository>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+            <id>cloudsoft-release</id>
+            <name>libs-release</name>
+            <url>https://artifactory.cloudsoftcorp.com/artifactory/libs-release-local</url>
+        </repository>
+        <repository>
+            <snapshots />
+            <id>cloudsoft-snapshots</id>
+            <name>libs-snapshot</name>
+            <url>https://artifactory.cloudsoftcorp.com/artifactory/libs-snapshot-local</url>
+        </repository>
+    </repositories>
+</project>

validation-testing/src/test/java/org/brooklyncentral/catalog/validation/testing/PullRequestValidationTest.java

@@ -0,0 +1,169 @@
+package org.brooklyncentral.catalog.validation.testing;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+import io.cloudsoft.catalog.util.CatalogValidator;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.OutputStream;
+import java.io.StringReader;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.eclipse.jgit.api.CloneCommand;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.diff.DiffEntry;
+import org.eclipse.jgit.diff.DiffFormatter;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectReader;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.revwalk.RevTree;
+import org.eclipse.jgit.revwalk.RevWalk;
+import org.eclipse.jgit.transport.RefSpec;
+import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
+import org.eclipse.jgit.treewalk.AbstractTreeIterator;
+import org.eclipse.jgit.treewalk.CanonicalTreeParser;
+import org.eclipse.jgit.treewalk.filter.PathFilter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.common.io.Files;
+
+public class PullRequestValidationTest {
+
+    private static final Logger LOG = LoggerFactory.getLogger(PullRequestValidationTest.class);
+
+    private static final String DIRECTORY_REPO_URI_ENV_VAR = "DIRECTORY_REPO_URI";
+    private static final String PR_NUMBER_ENV_VAR = "PR_NUMBER";
+    private static final String AUTH_TOKEN_ENV_VAR = "AUTH_TOKEN";
+
+    private static final String DEFAULT_REPOSITORIES_URI = "https://github.com/brooklyncentral/brooklyn-community-catalog";
+    private static final String FILE_TO_DIFF = "directory.yaml";
+    private static final String BRANCH_TO_TEST = "master";
+
+    private File cloneDirectory;
+
+    private Git git;
+
+    private String masterHash;
+    private String prHash;
+
+    @BeforeClass
+    public void setUp() throws Exception {
+        String repoUrl = Optional.fromNullable(System.getenv(DIRECTORY_REPO_URI_ENV_VAR)).or(DEFAULT_REPOSITORIES_URI);
+        String prNumber = System.getenv(PR_NUMBER_ENV_VAR);
+        String authToken = System.getenv(AUTH_TOKEN_ENV_VAR);
+
+        checkNotNull(prNumber, "PR number environment variable 'PR_NUMBER' must be set.");
+
+        cloneDirectory = Files.createTempDir();
+
+        CloneCommand cloneCommand = Git.cloneRepository()
+                .setDirectory(cloneDirectory)
+                .setURI(repoUrl)
+                .setBranchesToClone(ImmutableList.of("refs/heads/master"))
+                .setBranch("refs/heads/master");
+
+        if (authToken != null) {
+            cloneCommand = cloneCommand.setCredentialsProvider(new UsernamePasswordCredentialsProvider(authToken, ""));
+        }
+
+        git = cloneCommand.call();
+        git.fetch().setRefSpecs(new RefSpec(String.format("+refs/pull/%s/head:refs/remotes/origin/pr/%s", prNumber, prNumber))).call();
+
+        masterHash = git.getRepository().exactRef("refs/heads/master").getObjectId().getName();
+        prHash = git.getRepository().exactRef("refs/remotes/origin/pr/" + prNumber).getObjectId().getName();
+    }
+
+    @AfterClass
+    public void tearDown() throws Exception {
+        FileUtils.deleteDirectory(cloneDirectory);
+    }
+
+    @Test
+    public void validateNewRepos() throws Exception {
+        List<String> urlsToValidate = getURLsToValidate();
+
+        LOG.info("There are " + urlsToValidate.size() + " URL(s) to validate.");
+
+        for (String urlToValidate : urlsToValidate) {
+            LOG.info("Validating URL: '" + urlToValidate + "'");
+
+            try {
+                CatalogValidator catalogValidator = new CatalogValidator(urlToValidate, BRANCH_TO_TEST);
+                catalogValidator.validate();
+            } catch (Exception e) {
+                fail("Validation failed for URL: '" + urlToValidate + "'", e);
+            }
+
+            LOG.info("Successfully validated URL: '" + urlToValidate + "'");
+        }
+
+        LOG.info("All URL(s) successfully validated.");
+    }
+
+    private List<String> getURLsToValidate() throws Exception {
+        String diff = getDiff();
+        List<String> diffLines = IOUtils.readLines(new StringReader(diff));
+
+        List<String> urlsToValidate = new LinkedList<>();
+
+        for (String diffLine : diffLines) {
+            if (diffLine.startsWith("+-")) {
+                String urlToValidate = diffLine.replace("+-", "").trim();
+                urlsToValidate.add(urlToValidate);
+            }
+        }
+
+        return urlsToValidate;
+    }
+
+    private String getDiff() throws Exception {
+        AbstractTreeIterator oldTreeIterator = generateTreeIterator(git.getRepository(), masterHash);
+        AbstractTreeIterator newTreeIterator = generateTreeIterator(git.getRepository(), prHash);
+
+        List<DiffEntry> diff = git.diff()
+                .setOldTree(oldTreeIterator)
+                .setNewTree(newTreeIterator)
+                .setPathFilter(PathFilter.create(FILE_TO_DIFF))
+                .call();
+
+        assertTrue(diff.size() == 1, "Exactly one diff must match for file: '" + FILE_TO_DIFF + "'.");
+        DiffEntry entry = diff.get(0);
+
+        OutputStream diffOutputStream = new ByteArrayOutputStream();
+
+        try (DiffFormatter formatter = new DiffFormatter(diffOutputStream)) {
+            formatter.setRepository(git.getRepository());
+            formatter.format(entry);
+        }
+
+        return diffOutputStream.toString();
+    }
+
+    private static AbstractTreeIterator generateTreeIterator(Repository repository, String objectId) throws Exception {
+        try (RevWalk revWalk = new RevWalk(repository)) {
+            RevCommit revCommit = revWalk.parseCommit(ObjectId.fromString(objectId));
+            RevTree revTree = revWalk.parseTree(revCommit.getTree().getId());
+
+            CanonicalTreeParser canonicalTreeParser = new CanonicalTreeParser();
+            try (ObjectReader objectReader = repository.newObjectReader()) {
+                canonicalTreeParser.reset(objectReader, revTree.getId());
+            }
+
+            revWalk.dispose();
+
+            return canonicalTreeParser;
+        }
+    }
+}