[rebased, test WIP] i559 TexyServiceEngine

.gitignore

@@ -12,3 +12,5 @@
 build/
 .gradle
 .gradletasknamecache
+jbake-core/out/
+jbake-dist/out/

jbake-core/src/main/java/org/jbake/app/ContentStore.java

@@ -35,17 +35,16 @@
 import com.orientechnologies.orient.core.record.impl.ODocument;
 import com.orientechnologies.orient.core.sql.OCommandSQL;
 import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery;
-import org.jbake.model.DocumentAttributes;
-import org.jbake.model.DocumentTypes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.io.File;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import org.jbake.model.DocumentAttributes;
+import org.jbake.model.DocumentTypes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * @author jdlee
@@ -173,6 +172,36 @@ private void activateOnCurrentThread() {
         db.activateOnCurrentThread();
     }
 
+
+    /**
+     * Get a document by sourceUri and update it from the given map.
+     * @return the saved document.
+     * throws {@link IllegalArgumentException} if sourceUri or docType are null, or if the document doesn't exist.
+     */
+    public ODocument mergeDocument(Map<String, Object> incomingDocMap)
+    {
+        String sourceUri = (String) incomingDocMap.get(DocumentAttributes.SOURCE_URI.toString());
+        if (null == sourceUri)
+            throw new IllegalArgumentException("Document sourceUri is null.");
+        String docType = (String) incomingDocMap.get(Crawler.Attributes.TYPE);
+        if (null == docType)
+            throw new IllegalArgumentException("Document docType is null.");
+
+        // Get a document by sourceUri
+        String sql = "SELECT * FROM " + docType + " WHERE sourceuri=?";
+        activateOnCurrentThread();
+        List<ODocument> results = db.command(new OSQLSynchQuery<ODocument>(sql)).execute(sourceUri);
+        if (results.size() == 0)
+            throw new RuntimeException("No document with sourceUri '"+sourceUri+"'.");
+
+        // Update it from the given map.
+        ODocument incomingDoc = new ODocument(docType);
+        incomingDoc.fromMap(incomingDocMap);
+        ODocument merged = results.get(0).merge(incomingDoc, true, false);
+        return merged;
+    }
+
+
     public long getDocumentCount(String docType) {
         activateOnCurrentThread();
         return db.countClass(docType);
@@ -323,16 +352,27 @@ private void createDocType(final OSchema schema, final String docType) {
         logger.debug("Create document class '{}'", docType);
 
         OClass page = schema.createClass(docType);
-        page.createProperty(String.valueOf(DocumentAttributes.SHA1), OType.STRING).setNotNull(true);
-        page.createIndex(docType + "sha1Index", OClass.INDEX_TYPE.NOTUNIQUE, DocumentAttributes.SHA1.toString());
-        page.createProperty(String.valueOf(DocumentAttributes.SOURCE_URI), OType.STRING).setNotNull(true);
-        page.createIndex(docType + "sourceUriIndex", OClass.INDEX_TYPE.UNIQUE, DocumentAttributes.SOURCE_URI.toString());
-        page.createProperty(String.valueOf(DocumentAttributes.CACHED), OType.BOOLEAN).setNotNull(true);
-        page.createIndex(docType + "cachedIndex", OClass.INDEX_TYPE.NOTUNIQUE, DocumentAttributes.CACHED.toString());
-        page.createProperty(String.valueOf(DocumentAttributes.RENDERED), OType.BOOLEAN).setNotNull(true);
-        page.createIndex(docType + "renderedIndex", OClass.INDEX_TYPE.NOTUNIQUE, DocumentAttributes.RENDERED.toString());
-        page.createProperty(String.valueOf(DocumentAttributes.STATUS), OType.STRING).setNotNull(true);
-        page.createIndex(docType + "statusIndex", OClass.INDEX_TYPE.NOTUNIQUE, DocumentAttributes.STATUS.toString());
+
+        // Primary key
+        String attribName = DocumentAttributes.SOURCE_URI.toString();
+        page.createProperty(attribName, OType.STRING).setNotNull(true);
+        page.createIndex(docType + "sourceUriIndex", OClass.INDEX_TYPE.UNIQUE, attribName);
+
+        attribName = DocumentAttributes.SHA1.toString();
+        page.createProperty(attribName, OType.STRING).setNotNull(true);
+        page.createIndex(docType + "sha1Index", OClass.INDEX_TYPE.NOTUNIQUE, attribName);
+
+        attribName = DocumentAttributes.CACHED.toString();
+        page.createProperty(attribName, OType.BOOLEAN).setNotNull(true);
+        page.createIndex(docType + "cachedIndex", OClass.INDEX_TYPE.NOTUNIQUE, attribName);
+
+        attribName = DocumentAttributes.RENDERED.toString();
+        page.createProperty(attribName, OType.BOOLEAN).setNotNull(true);
+        page.createIndex(docType + "renderedIndex", OClass.INDEX_TYPE.NOTUNIQUE, attribName);
+
+        attribName = DocumentAttributes.STATUS.toString();
+        page.createProperty(attribName, OType.STRING).setNotNull(true);
+        page.createIndex(docType + "statusIndex", OClass.INDEX_TYPE.NOTUNIQUE, attribName);
     }
 
     private void createSignatureType(OSchema schema) {

jbake-core/src/main/java/org/jbake/app/Crawler.java

@@ -1,8 +1,17 @@
 package org.jbake.app;
 
 import com.orientechnologies.orient.core.record.impl.ODocument;
+import java.io.File;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.Map;
 import org.apache.commons.configuration.CompositeConfiguration;
 import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang3.BooleanUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.jbake.app.Crawler.Attributes.Status;
 import org.jbake.app.configuration.JBakeConfiguration;
 import org.jbake.app.configuration.JBakeConfigurationFactory;
@@ -13,14 +22,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
-import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.Map;
-
 /**
  * Crawls a file system looking for content.
  *
@@ -29,18 +30,19 @@
 public class Crawler {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(Crawler.class);
+
     public static final String URI_SEPARATOR_CHAR = "/";
     private final ContentStore db;
     private JBakeConfiguration config;
     private Parser parser;
 
     /**
+     * Creates new instance of Crawler.
+     *
      * @param db     Database instance for content
      * @param source Base directory where content directory is located
      * @param config Project configuration
      * @deprecated Use {@link #Crawler(ContentStore, JBakeConfiguration)} instead.
-     * <p>
-     * Creates new instance of Crawler.
      */
     @Deprecated
     public Crawler(ContentStore db, File source, CompositeConfiguration config) {
@@ -93,13 +95,14 @@ private void crawl(File path) {
                     DocumentStatus status = DocumentStatus.NEW;
                     for (String docType : DocumentTypes.getDocumentTypes()) {
                         status = findDocumentStatus(docType, uri, sha1);
-                        if (status == DocumentStatus.UPDATED) {
-                            sb.append(" : modified ");
-                            db.deleteContent(docType, uri);
-
-                        } else if (status == DocumentStatus.IDENTICAL) {
-                            sb.append(" : same ");
-                            process = false;
+                        switch (status) {
+                            case UPDATED:
+                                sb.append(" : modified ");
+                                db.deleteContent(docType, uri);
+                                break;
+                            case IDENTICAL:
+                                sb.append(" : same ");
+                                process = false;
                         }
                         if (!process) {
                             break;
@@ -131,50 +134,43 @@ private String buildHash(final File sourceFile) {
         return sha1;
     }
 
-    private String buildURI(final File sourceFile) {
-        String uri = FileUtil.asPath(sourceFile).replace(FileUtil.asPath(config.getContentFolder()), "");
+    private String buildURI(final File sourceFile)
+    {
+        String contentPath = FileUtil.asPath(config.getContentFolder());
+
+        //  /jbake-web/content/path/to/file.ext
+        //                ->   path/to/file.ext
+        String uri = FileUtil.asPath(sourceFile).replace(contentPath, "");
 
         // On windows we have to replace the backslash
         if (!File.separator.equals(URI_SEPARATOR_CHAR)) {
             uri = uri.replace(File.separator, URI_SEPARATOR_CHAR);
         }
 
-        if (useNoExtensionUri(uri)) {
-            // convert URI from xxx.html to xxx/index.html
-            uri = createNoExtensionUri(uri);
-        } else {
-            uri = createUri(uri);
-        }
+        uri = createUri(uri, useNoExtensionUri(uri));
 
-        // strip off leading / to enable generating non-root based sites
-        if (uri.startsWith(URI_SEPARATOR_CHAR)) {
-            uri = uri.substring(1, uri.length());
-        }
+        // Strip off leading / to enable generating non-root based sites
+        uri = StringUtils.removeStart(uri, URI_SEPARATOR_CHAR);
 
         return uri;
     }
 
-    // TODO: Refactor - parametrize the following two methods into one.
-    // commons-codec's URLCodec could be used when we add that dependency.
-    private String createUri(String uri) {
+    /**
+     * Takes care of file name characters that need to be URL-encoded.
+     * TODO: Take care of URL-encoding for directory names.
+     * @param extensionlessMode  If true, converts URI from path/to/file.html to path/to/file/index.html,
+     *                           so that the user URL can be "path/to/file".
+     */
+    private String createUri(String uri, boolean extensionlessMode) {
         try {
-            return URI_SEPARATOR_CHAR + FilenameUtils.getPath(uri)
+            return "/" + FilenameUtils.getPath(uri)
+                    // commons-codec's URLCodec could be used when we add that dependency.
                     + URLEncoder.encode(FilenameUtils.getBaseName(uri), StandardCharsets.UTF_8.name())
+                    // If asked to, convert URI from xxx.html to xxx/index.html
+                    + (extensionlessMode ? URI_SEPARATOR_CHAR + "index" : "")
                     + config.getOutputExtension();
-        } catch (UnsupportedEncodingException e) {
-            throw new RuntimeException("Missing UTF-8 encoding??", e); // Won't happen unless JDK is broken.
         }
-    }
-
-    private String createNoExtensionUri(String uri) {
-        try {
-            return URI_SEPARATOR_CHAR
-                    + FilenameUtils.getPath(uri)
-                    + URLEncoder.encode(FilenameUtils.getBaseName(uri), StandardCharsets.UTF_8.name())
-                    + URI_SEPARATOR_CHAR
-                    + "index"
-                    + config.getOutputExtension();
-        } catch (UnsupportedEncodingException e) {
+        catch (UnsupportedEncodingException e) {
             throw new RuntimeException("Missing UTF-8 encoding??", e); // Won't happen unless JDK is broken.
         }
     }
@@ -190,45 +186,54 @@ private boolean useNoExtensionUri(String uri) {
     }
 
     private void crawlSourceFile(final File sourceFile, final String sha1, final String uri) {
-        Map<String, Object> fileContents = parser.processFile(sourceFile);
-        if (fileContents != null) {
-            fileContents.put(Attributes.ROOTPATH, getPathToRoot(sourceFile));
-            fileContents.put(String.valueOf(DocumentAttributes.SHA1), sha1);
-            fileContents.put(String.valueOf(DocumentAttributes.RENDERED), false);
-            if (fileContents.get(Attributes.TAGS) != null) {
-                // store them as a String[]
-                String[] tags = (String[]) fileContents.get(Attributes.TAGS);
-                fileContents.put(Attributes.TAGS, tags);
-            }
-            fileContents.put(Attributes.FILE, sourceFile.getPath());
-            fileContents.put(String.valueOf(DocumentAttributes.SOURCE_URI), uri);
-            fileContents.put(Attributes.URI, uri);
-
-            String documentType = (String) fileContents.get(Attributes.TYPE);
-            if (fileContents.get(Attributes.STATUS).equals(Status.PUBLISHED_DATE)) {
-                if (fileContents.get(Attributes.DATE) != null && (fileContents.get(Attributes.DATE) instanceof Date)) {
-                    if (new Date().after((Date) fileContents.get(Attributes.DATE))) {
-                        fileContents.put(Attributes.STATUS, Status.PUBLISHED);
+        try {
+            Map<String, Object> fileContents = parser.processFile(sourceFile);
+            if (fileContents != null) {
+                fileContents.put(Attributes.ROOTPATH, getPathToRoot(sourceFile));
+                fileContents.put(String.valueOf(DocumentAttributes.SHA1), sha1);
+                fileContents.put(String.valueOf(DocumentAttributes.RENDERED), false);
+                if (fileContents.get(Attributes.TAGS) != null) {
+                    // store them as a String[]
+                    String[] tags = (String[]) fileContents.get(Attributes.TAGS);
+                    fileContents.put(Attributes.TAGS, tags);
+                }
+                fileContents.put(Attributes.FILE, sourceFile.getPath());
+                fileContents.put(String.valueOf(DocumentAttributes.SOURCE_URI), uri);
+                fileContents.put(Attributes.URI, uri);
+
+                String documentType = (String) fileContents.get(Attributes.TYPE);
+                if (fileContents.get(Attributes.STATUS).equals(Status.PUBLISHED_DATE)) {
+                    if (fileContents.get(Attributes.DATE) != null && (fileContents.get(Attributes.DATE) instanceof Date)) {
+                        if (new Date().after((Date) fileContents.get(Attributes.DATE))) {
+                            fileContents.put(Attributes.STATUS, Status.PUBLISHED);
+                        }
                     }
                 }
-            }
 
-            if (config.getUriWithoutExtension()) {
-                fileContents.put(Attributes.NO_EXTENSION_URI, uri.replace("/index.html", "/"));
-            }
+                if (config.getUriWithoutExtension()) {
+                    fileContents.put(Attributes.NO_EXTENSION_URI, uri.replace("/index.html", "/"));
+                }
 
-            if (config.getImgPathUpdate()) {
-                // Prevent image source url's from breaking
-                HtmlUtil.fixImageSourceUrls(fileContents, config);
-            }
+                if (config.getImgPathUpdate()) {
+                    // Prevent image source url's from breaking
+                    HtmlUtil.fixImageSourceUrls(fileContents, config);
+                }
 
-            ODocument doc = new ODocument(documentType);
-            doc.fromMap(fileContents);
-            boolean cached = fileContents.get(String.valueOf(DocumentAttributes.CACHED)) != null ? Boolean.valueOf((String) fileContents.get(String.valueOf(DocumentAttributes.CACHED))) : true;
-            doc.field(String.valueOf(DocumentAttributes.CACHED), cached);
-            doc.save();
-        } else {
-            LOGGER.warn("{} has an invalid header, it has been ignored!", sourceFile);
+                ODocument doc = new ODocument(documentType);
+                doc.fromMap(fileContents);
+                // This just stores true if it's not there.
+                String attrCached = DocumentAttributes.CACHED.toString();
+                String currentValue = (String) fileContents.get(attrCached);
+                boolean isCached = BooleanUtils.toBoolean(currentValue, null, "false");
+                doc.field(attrCached, isCached);
+                doc.save();
+            }
+            else {
+                LOGGER.warn("{} has an invalid header, it has been ignored!", sourceFile);
+            }
+        }
+        catch (Exception ex) {
+            throw new RuntimeException("Failed crawling file: " + sourceFile.getPath() + " " + ex.getMessage(), ex);
         }
     }
 

jbake-core/src/main/java/org/jbake/app/DebugUtil.java

@@ -0,0 +1,30 @@
+package org.jbake.app;
+
+import java.io.PrintStream;
+import java.sql.SQLOutput;
+import java.util.Map;
+import org.apache.commons.lang.StringUtils;
+
+public class DebugUtil
+{
+    public static <T extends Object> void printMap(String label, Map<String, T> map, PrintStream printStream){
+        if (null == map) {
+            printStream.println("The Map is null.");
+            return;
+        }
+
+        if (null == printStream) {
+            printStream = System.out;
+            return;
+        }
+        if (!StringUtils.isBlank(label)) {
+            printStream.println(label + ":");
+        }
+
+        printStream.println();
+        for (Map.Entry<String, T> entry: map.entrySet()) {
+            printStream.println(entry.getKey() + " :: " + entry.getValue());
+        }
+        printStream.println();
+    }
+}