From cce1dd373a54772df7cc76ecd8f2ceb32bd6a004 Mon Sep 17 00:00:00 2001 From: Ondrej Zizka Date: Mon, 23 Jul 2018 17:38:25 +0200 Subject: [PATCH 1/6] Add Texy markup engine imports cleanup. Final touches. Extract the title from the Texy documents. Improve hasHeader(): only scan first N lines, skip #... lines, skip blank lines, test against a regex; do not require status and type if defaults are set. Add DebugUtil with map printing, since JBake code uses maps heavily Wrap Crawler iteration into a try/catch Start refactoring MarkupEngine so it supports files without a header. Make DebugUtil more generic Improve MarkupEngine: Support no header if all values are known; Improve validation; Refactor. Extract title from Texy documents. Implement RawMarkupEngine: Extract title from the HTML; Normalize HTML; Pretty print HTML; Export as XHTML; Change exported charset; Introduce `input.charset`. Fix MarkupEngine - don't return headers map if the header separator is not found. Allow .-_ in the header names Refactor HtmlUtil#fixImageSourceUrls(). Keeps the same behavior. Fix https://github.com/jbake-org/jbake/issues/499 file names encoding. Implements https://github.com/jbake-org/jbake/issues/500 Make URL fixing optional #500 Refactor createUri() and createNoExtensionUri() into one. Make index creation bit more readable (just reorder) Make index creation bit more readable (reuse the attrib name) Refactor Crawler#crawlSourceFile() logic around updating cache flag. Implement ContentStore#mergeDocument(Map) to update docs. Implement Make "relative points to assets" optional #502 Implement Make URL fixing optional #500 These two are hitting the same code, so it's hard to split them. MarkupUtil and RawMarkupUtil cleanup DebugUtil call Force normalize HTML files if they contain . Rename vars Allow deduplication of title autodetected from the document's header - mark that header with a CSS class. Fix: Storing the altered DOM wrapped in
resulted in this
being serialized too. This removes it. Make innerXml more robust. --- .../main/java/org/jbake/app/ConfigUtil.java | 0 .../main/java/org/jbake/app/ContentStore.java | 70 ++++- .../src/main/java/org/jbake/app/Crawler.java | 165 +++++------ .../main/java/org/jbake/app/DebugUtil.java | 15 + .../src/main/java/org/jbake/app/Oven.java | 96 ++++++- .../DefaultJBakeConfiguration.java | 67 +++++ .../app/configuration/JBakeConfiguration.java | 44 +++ .../app/configuration/JBakeProperty.java | 10 + .../org/jbake/parser/AsciidoctorEngine.java | 2 +- .../java/org/jbake/parser/ErrorEngine.java | 4 +- .../java/org/jbake/parser/MarkupEngine.java | 258 +++++++++--------- .../org/jbake/parser/RawMarkupEngine.java | 132 +++++++++ .../jbake/parser/texy/TexyRestService.java | 50 ++++ .../jbake/parser/texy/TexyServiceEngine.java | 69 +++++ .../org/jbake/parser/texy/TitleExtractor.java | 112 ++++++++ .../org/jbake/render/ArchiveRenderer.java | 5 +- .../org/jbake/render/BaseRenderingTool.java | 10 + .../org/jbake/render/DocumentsRenderer.java | 44 +-- .../java/org/jbake/render/FeedRenderer.java | 6 +- .../java/org/jbake/render/IndexRenderer.java | 7 +- .../java/org/jbake/render/RenderingTool.java | 10 +- .../org/jbake/render/SitemapRenderer.java | 7 +- .../java/org/jbake/render/TagsRenderer.java | 7 +- .../main/java/org/jbake/util/HtmlUtil.java | 79 ++++-- .../org.jbake.parser.MarkupEngines.properties | 3 +- .../src/main/resources/default.properties | 14 +- jbake-dist/src/dist/lib/logging/logback.xml | 1 + 27 files changed, 981 insertions(+), 306 deletions(-) delete mode 100644 jbake-core/src/main/java/org/jbake/app/ConfigUtil.java create mode 100644 jbake-core/src/main/java/org/jbake/app/DebugUtil.java create mode 100644 jbake-core/src/main/java/org/jbake/parser/texy/TexyRestService.java create mode 100644 jbake-core/src/main/java/org/jbake/parser/texy/TexyServiceEngine.java create mode 100644 jbake-core/src/main/java/org/jbake/parser/texy/TitleExtractor.java create mode 100644 jbake-core/src/main/java/org/jbake/render/BaseRenderingTool.java diff --git a/jbake-core/src/main/java/org/jbake/app/ConfigUtil.java b/jbake-core/src/main/java/org/jbake/app/ConfigUtil.java deleted file mode 100644 index e69de29bb..000000000 diff --git a/jbake-core/src/main/java/org/jbake/app/ContentStore.java b/jbake-core/src/main/java/org/jbake/app/ContentStore.java index f24ba540d..6ddae6523 100644 --- a/jbake-core/src/main/java/org/jbake/app/ContentStore.java +++ b/jbake-core/src/main/java/org/jbake/app/ContentStore.java @@ -35,17 +35,16 @@ import com.orientechnologies.orient.core.record.impl.ODocument; import com.orientechnologies.orient.core.sql.OCommandSQL; import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery; -import org.jbake.model.DocumentAttributes; -import org.jbake.model.DocumentTypes; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import org.jbake.model.DocumentAttributes; +import org.jbake.model.DocumentTypes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @author jdlee @@ -173,6 +172,36 @@ private void activateOnCurrentThread() { db.activateOnCurrentThread(); } + + /** + * Get a document by sourceUri and update it from the given map. + * @return the saved document. + * @throws Exception if sourceUri or docType are null, or if the document doesn't exist. + */ + public ODocument mergeDocument(Map incomingDocMap) + { + String sourceUri = (String) incomingDocMap.get(DocumentAttributes.SOURCE_URI.toString()); + if (null == sourceUri) + throw new IllegalArgumentException("Document sourceUri is null."); + String docType = (String) incomingDocMap.get(Crawler.Attributes.TYPE); + if (null == docType) + throw new IllegalArgumentException("Document docType is null."); + + // Get a document by sourceUri + String sql = "SELECT * FROM " + docType + " WHERE sourceuri=?"; + activateOnCurrentThread(); + List results = db.command(new OSQLSynchQuery(sql)).execute(sourceUri); + if (results.size() == 0) + throw new RuntimeException("No document with sourceUri '"+sourceUri+"'."); + + // Update it from the given map. + ODocument incomingDoc = new ODocument(docType); + incomingDoc.fromMap(incomingDocMap); + ODocument merged = results.get(0).merge(incomingDoc, true, false); + return merged; + } + + public long getDocumentCount(String docType) { activateOnCurrentThread(); return db.countClass(docType); @@ -323,16 +352,27 @@ private void createDocType(final OSchema schema, final String docType) { logger.debug("Create document class '{}'", docType); OClass page = schema.createClass(docType); - page.createProperty(String.valueOf(DocumentAttributes.SHA1), OType.STRING).setNotNull(true); - page.createIndex(docType + "sha1Index", OClass.INDEX_TYPE.NOTUNIQUE, DocumentAttributes.SHA1.toString()); - page.createProperty(String.valueOf(DocumentAttributes.SOURCE_URI), OType.STRING).setNotNull(true); - page.createIndex(docType + "sourceUriIndex", OClass.INDEX_TYPE.UNIQUE, DocumentAttributes.SOURCE_URI.toString()); - page.createProperty(String.valueOf(DocumentAttributes.CACHED), OType.BOOLEAN).setNotNull(true); - page.createIndex(docType + "cachedIndex", OClass.INDEX_TYPE.NOTUNIQUE, DocumentAttributes.CACHED.toString()); - page.createProperty(String.valueOf(DocumentAttributes.RENDERED), OType.BOOLEAN).setNotNull(true); - page.createIndex(docType + "renderedIndex", OClass.INDEX_TYPE.NOTUNIQUE, DocumentAttributes.RENDERED.toString()); - page.createProperty(String.valueOf(DocumentAttributes.STATUS), OType.STRING).setNotNull(true); - page.createIndex(docType + "statusIndex", OClass.INDEX_TYPE.NOTUNIQUE, DocumentAttributes.STATUS.toString()); + + // Primary key + String attribName = DocumentAttributes.SOURCE_URI.toString(); + page.createProperty(attribName, OType.STRING).setNotNull(true); + page.createIndex(docType + "sourceUriIndex", OClass.INDEX_TYPE.UNIQUE, attribName); + + attribName = DocumentAttributes.SHA1.toString(); + page.createProperty(attribName, OType.STRING).setNotNull(true); + page.createIndex(docType + "sha1Index", OClass.INDEX_TYPE.NOTUNIQUE, attribName); + + attribName = DocumentAttributes.CACHED.toString(); + page.createProperty(attribName, OType.BOOLEAN).setNotNull(true); + page.createIndex(docType + "cachedIndex", OClass.INDEX_TYPE.NOTUNIQUE, attribName); + + attribName = DocumentAttributes.RENDERED.toString(); + page.createProperty(attribName, OType.BOOLEAN).setNotNull(true); + page.createIndex(docType + "renderedIndex", OClass.INDEX_TYPE.NOTUNIQUE, attribName); + + attribName = DocumentAttributes.STATUS.toString(); + page.createProperty(attribName, OType.STRING).setNotNull(true); + page.createIndex(docType + "statusIndex", OClass.INDEX_TYPE.NOTUNIQUE, attribName); } private void createSignatureType(OSchema schema) { diff --git a/jbake-core/src/main/java/org/jbake/app/Crawler.java b/jbake-core/src/main/java/org/jbake/app/Crawler.java index 617974dbf..d6e642518 100644 --- a/jbake-core/src/main/java/org/jbake/app/Crawler.java +++ b/jbake-core/src/main/java/org/jbake/app/Crawler.java @@ -1,8 +1,17 @@ package org.jbake.app; import com.orientechnologies.orient.core.record.impl.ODocument; +import java.io.File; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Date; +import java.util.Map; import org.apache.commons.configuration.CompositeConfiguration; import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; import org.jbake.app.Crawler.Attributes.Status; import org.jbake.app.configuration.JBakeConfiguration; import org.jbake.app.configuration.JBakeConfigurationFactory; @@ -13,14 +22,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.Date; -import java.util.Map; - /** * Crawls a file system looking for content. * @@ -29,18 +30,19 @@ public class Crawler { private static final Logger LOGGER = LoggerFactory.getLogger(Crawler.class); + public static final String URI_SEPARATOR_CHAR = "/"; private final ContentStore db; private JBakeConfiguration config; private Parser parser; /** + * Creates new instance of Crawler. + * * @param db Database instance for content * @param source Base directory where content directory is located * @param config Project configuration * @deprecated Use {@link #Crawler(ContentStore, JBakeConfiguration)} instead. - *

- * Creates new instance of Crawler. */ @Deprecated public Crawler(ContentStore db, File source, CompositeConfiguration config) { @@ -93,13 +95,14 @@ private void crawl(File path) { DocumentStatus status = DocumentStatus.NEW; for (String docType : DocumentTypes.getDocumentTypes()) { status = findDocumentStatus(docType, uri, sha1); - if (status == DocumentStatus.UPDATED) { - sb.append(" : modified "); - db.deleteContent(docType, uri); - - } else if (status == DocumentStatus.IDENTICAL) { - sb.append(" : same "); - process = false; + switch (status) { + case UPDATED: + sb.append(" : modified "); + db.deleteContent(docType, uri); + break; + case IDENTICAL: + sb.append(" : same "); + process = false; } if (!process) { break; @@ -131,50 +134,43 @@ private String buildHash(final File sourceFile) { return sha1; } - private String buildURI(final File sourceFile) { - String uri = FileUtil.asPath(sourceFile).replace(FileUtil.asPath(config.getContentFolder()), ""); + private String buildURI(final File sourceFile) + { + String contentPath = FileUtil.asPath(config.getContentFolder()); + + // /jbake-web/content/path/to/file.ext + // -> path/to/file.ext + String uri = FileUtil.asPath(sourceFile).replace(contentPath, ""); // On windows we have to replace the backslash if (!File.separator.equals(URI_SEPARATOR_CHAR)) { uri = uri.replace(File.separator, URI_SEPARATOR_CHAR); } - if (useNoExtensionUri(uri)) { - // convert URI from xxx.html to xxx/index.html - uri = createNoExtensionUri(uri); - } else { - uri = createUri(uri); - } + uri = createUri(uri, useNoExtensionUri(uri)); - // strip off leading / to enable generating non-root based sites - if (uri.startsWith(URI_SEPARATOR_CHAR)) { - uri = uri.substring(1, uri.length()); - } + // Strip off leading / to enable generating non-root based sites + uri = StringUtils.removeStart(uri, URI_SEPARATOR_CHAR); return uri; } - // TODO: Refactor - parametrize the following two methods into one. - // commons-codec's URLCodec could be used when we add that dependency. - private String createUri(String uri) { + /** + * Takes care of file name characters that need to be URL-encoded. + * TODO: Take care of URL-encoding for directory names. + * @param extensionlessMode If true, converts URI from path/to/file.html to path/to/file/index.html, + * so that the user URL can be "path/to/file". + */ + private String createUri(String uri, boolean extensionlessMode) { try { - return URI_SEPARATOR_CHAR + FilenameUtils.getPath(uri) + return "/" + FilenameUtils.getPath(uri) + // commons-codec's URLCodec could be used when we add that dependency. + URLEncoder.encode(FilenameUtils.getBaseName(uri), StandardCharsets.UTF_8.name()) + // If asked to, convert URI from xxx.html to xxx/index.html + + (extensionlessMode ? URI_SEPARATOR_CHAR + "index" : "") + config.getOutputExtension(); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("Missing UTF-8 encoding??", e); // Won't happen unless JDK is broken. } - } - - private String createNoExtensionUri(String uri) { - try { - return URI_SEPARATOR_CHAR - + FilenameUtils.getPath(uri) - + URLEncoder.encode(FilenameUtils.getBaseName(uri), StandardCharsets.UTF_8.name()) - + URI_SEPARATOR_CHAR - + "index" - + config.getOutputExtension(); - } catch (UnsupportedEncodingException e) { + catch (UnsupportedEncodingException e) { throw new RuntimeException("Missing UTF-8 encoding??", e); // Won't happen unless JDK is broken. } } @@ -190,45 +186,54 @@ private boolean useNoExtensionUri(String uri) { } private void crawlSourceFile(final File sourceFile, final String sha1, final String uri) { - Map fileContents = parser.processFile(sourceFile); - if (fileContents != null) { - fileContents.put(Attributes.ROOTPATH, getPathToRoot(sourceFile)); - fileContents.put(String.valueOf(DocumentAttributes.SHA1), sha1); - fileContents.put(String.valueOf(DocumentAttributes.RENDERED), false); - if (fileContents.get(Attributes.TAGS) != null) { - // store them as a String[] - String[] tags = (String[]) fileContents.get(Attributes.TAGS); - fileContents.put(Attributes.TAGS, tags); - } - fileContents.put(Attributes.FILE, sourceFile.getPath()); - fileContents.put(String.valueOf(DocumentAttributes.SOURCE_URI), uri); - fileContents.put(Attributes.URI, uri); - - String documentType = (String) fileContents.get(Attributes.TYPE); - if (fileContents.get(Attributes.STATUS).equals(Status.PUBLISHED_DATE)) { - if (fileContents.get(Attributes.DATE) != null && (fileContents.get(Attributes.DATE) instanceof Date)) { - if (new Date().after((Date) fileContents.get(Attributes.DATE))) { - fileContents.put(Attributes.STATUS, Status.PUBLISHED); + try { + Map fileContents = parser.processFile(sourceFile); + if (fileContents != null) { + fileContents.put(Attributes.ROOTPATH, getPathToRoot(sourceFile)); + fileContents.put(String.valueOf(DocumentAttributes.SHA1), sha1); + fileContents.put(String.valueOf(DocumentAttributes.RENDERED), false); + if (fileContents.get(Attributes.TAGS) != null) { + // store them as a String[] + String[] tags = (String[]) fileContents.get(Attributes.TAGS); + fileContents.put(Attributes.TAGS, tags); + } + fileContents.put(Attributes.FILE, sourceFile.getPath()); + fileContents.put(String.valueOf(DocumentAttributes.SOURCE_URI), uri); + fileContents.put(Attributes.URI, uri); + + String documentType = (String) fileContents.get(Attributes.TYPE); + if (fileContents.get(Attributes.STATUS).equals(Status.PUBLISHED_DATE)) { + if (fileContents.get(Attributes.DATE) != null && (fileContents.get(Attributes.DATE) instanceof Date)) { + if (new Date().after((Date) fileContents.get(Attributes.DATE))) { + fileContents.put(Attributes.STATUS, Status.PUBLISHED); + } } } - } - if (config.getUriWithoutExtension()) { - fileContents.put(Attributes.NO_EXTENSION_URI, uri.replace("/index.html", "/")); - } + if (config.getUriWithoutExtension()) { + fileContents.put(Attributes.NO_EXTENSION_URI, uri.replace("/index.html", "/")); + } - if (config.getImgPathUpdate()) { - // Prevent image source url's from breaking - HtmlUtil.fixImageSourceUrls(fileContents, config); - } + if (config.getImgPathUpdate()) { + // Prevent image source url's from breaking + HtmlUtil.fixImageSourceUrls(fileContents, config); + } - ODocument doc = new ODocument(documentType); - doc.fromMap(fileContents); - boolean cached = fileContents.get(String.valueOf(DocumentAttributes.CACHED)) != null ? Boolean.valueOf((String) fileContents.get(String.valueOf(DocumentAttributes.CACHED))) : true; - doc.field(String.valueOf(DocumentAttributes.CACHED), cached); - doc.save(); - } else { - LOGGER.warn("{} has an invalid header, it has been ignored!", sourceFile); + ODocument doc = new ODocument(documentType); + doc.fromMap(fileContents); + // This just stores true if it's not there. + String attrCached = DocumentAttributes.CACHED.toString(); + String currentValue = (String) fileContents.get(attrCached); + boolean isCached = BooleanUtils.toBoolean(currentValue, null, "false"); + doc.field(attrCached, isCached); + doc.save(); + } + else { + LOGGER.warn("{} has an invalid header, it has been ignored!", sourceFile); + } + } + catch (Exception ex) { + throw new RuntimeException("Failed crawling file: " + sourceFile.getPath() + " " + ex.getMessage(), ex); } } diff --git a/jbake-core/src/main/java/org/jbake/app/DebugUtil.java b/jbake-core/src/main/java/org/jbake/app/DebugUtil.java new file mode 100644 index 000000000..001169930 --- /dev/null +++ b/jbake-core/src/main/java/org/jbake/app/DebugUtil.java @@ -0,0 +1,15 @@ +package org.jbake.app; + +import java.io.PrintStream; +import java.util.Map; + +public class DebugUtil +{ + public static void printMap(Map map, PrintStream printStream){ + printStream.println(); + for (Map.Entry entry: map.entrySet()) { + printStream.println(entry.getKey() + " :: " + entry.getValue()); + } + printStream.println(); + } +} diff --git a/jbake-core/src/main/java/org/jbake/app/Oven.java b/jbake-core/src/main/java/org/jbake/app/Oven.java index 9eb9a6781..7842545dc 100644 --- a/jbake-core/src/main/java/org/jbake/app/Oven.java +++ b/jbake-core/src/main/java/org/jbake/app/Oven.java @@ -1,5 +1,12 @@ package org.jbake.app; +import java.io.File; +import java.util.ArrayList; +import java.util.Date; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.ServiceLoader; import org.apache.commons.configuration.CompositeConfiguration; import org.jbake.app.configuration.DefaultJBakeConfiguration; import org.jbake.app.configuration.JBakeConfiguration; @@ -10,16 +17,10 @@ import org.jbake.template.ModelExtractors; import org.jbake.template.ModelExtractorsDocumentTypeListener; import org.jbake.template.RenderingException; +import org.jbake.util.HtmlUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.util.ArrayList; -import java.util.Date; -import java.util.LinkedList; -import java.util.List; -import java.util.ServiceLoader; - /** * All the baking happens in the Oven! * @@ -139,11 +140,13 @@ public void bake() { // render content renderContent(); - // copy assets - asset.copy(); - asset.copyAssetsFromContent(config.getContentFolder()); + { + // copy assets + asset.copy(); + asset.copyAssetsFromContent(config.getContentFolder()); - errors.addAll(asset.getErrors()); + errors.addAll(asset.getErrors()); + } LOGGER.info("Baking finished!"); long end = new Date().getTime(); @@ -157,6 +160,40 @@ public void bake() { } } + /** + * Replaces the URLs to resources in documents so that they are pointing to the resources even if the rendered document is placed + * somewhere else than the source markup. + */ + private void makeUrlsAbsolute(ContentStore db, CompositeConfiguration config) + { + int renderedCount = 0; + final List errors = new LinkedList(); + for (String docType : DocumentTypes.getDocumentTypes()) { + DocumentList documentList = db.getUnrenderedContent(docType); + if(documentList == null) continue; + + for (Map documentMap : documentList) { + try { + HtmlUtil.fixImageSourceUrls(documentMap, config); + // Save + documentMap = db.mergeDocument(documentMap).toMap(); + } + catch (Exception e) { + errors.add(e.getMessage()); + } + } + } + + if (!errors.isEmpty()) { + StringBuilder sb = new StringBuilder(); + sb.append("Failed to render documents. Cause(s):"); + for (String error : errors) { + sb.append("\n ").append(error); + } + throw new RuntimeException(sb.toString()); + } + } + /** * Iterates over the configuration, searching for keys like "template.index.file=..." * in order to register new document types. @@ -186,13 +223,46 @@ private void renderContent() { Renderer renderer = utensils.getRenderer(); ContentStore contentStore = utensils.getContentStore(); - for (RenderingTool tool : ServiceLoader.load(RenderingTool.class)) { + ServiceLoader renderTools = ServiceLoader.load(RenderingTool.class); + + // If this is enabled, then this already happened in Crawler. + // TODO: Remove the fixing from Crawler. + // We should keep the pristine doc body as long as possible, or change it locally. + boolean fixedAlready = config.getMakeImagesUrlAbolute(); + + // 1st pass without altered URLs. + for (RenderingTool tool : renderTools) { + if (!tool.isRendersInPlace() || fixedAlready) + continue; try { renderedCount += tool.render(renderer, contentStore, config); - } catch (RenderingException e) { + } + catch (RenderingException e) { errors.add(e); } } + + // Make the URLs absolute. + if (!fixedAlready) { + makeUrlsAbsolute(contentStore, config); + + // 2nd pass with absolutized URLs. + for (RenderingTool tool : renderTools) { + if (tool.isRendersInPlace()) + continue; + try { + renderedCount += tool.render(renderer, contentStore, config); + } + catch (RenderingException e) { + errors.add(e); + } + } + } + + // mark docs as rendered + for (String docType : DocumentTypes.getDocumentTypes()) { + contentStore.markContentAsRendered(docType); + } } diff --git a/jbake-core/src/main/java/org/jbake/app/configuration/DefaultJBakeConfiguration.java b/jbake-core/src/main/java/org/jbake/app/configuration/DefaultJBakeConfiguration.java index 7a772acb8..5c850ef54 100644 --- a/jbake-core/src/main/java/org/jbake/app/configuration/DefaultJBakeConfiguration.java +++ b/jbake-core/src/main/java/org/jbake/app/configuration/DefaultJBakeConfiguration.java @@ -1,8 +1,11 @@ package org.jbake.app.configuration; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import org.apache.commons.configuration.CompositeConfiguration; import org.apache.commons.configuration.Configuration; import org.apache.commons.lang3.StringUtils; +import org.jbake.app.JBakeException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -468,6 +471,69 @@ public String getVersion() { return getAsString(JBakeProperty.VERSION); } + + @Override + public boolean getExtractTitleFromDoc() + { + return getAsBoolean(JBakeProperty.EXTRACT_TITLE_FROM_DOC); + } + + @Override + public boolean getNormalizeHtml() + { + return getAsBoolean(JBakeProperty.HTML_NORMALIZE); + } + + @Override + public boolean getConvertHtmlToXhtml() + { + return getAsBoolean(JBakeProperty.HTML_CONVERT_TO_XHTML); + } + + @Override + public boolean getPrettyPrintHtml() + { + return getAsBoolean(JBakeProperty.HTML_PRETTY_PRINT); + } + + @Override + public Charset getOutputHtmlCharset() + { + String charsetStr = getAsString(JBakeProperty.HTML_OUTPUT_CHARSET); + try { + return Charset.forName(charsetStr); + } + catch (Exception ex) { + throw new JBakeException("Unknown character set: " + charsetStr + " Try " + StandardCharsets.UTF_8.name()); + } + } + + @Override + public Charset getInputCharset() + { + String charsetStr = getAsString(JBakeProperty.INPUT_CHARSET); + try { + return Charset.forName(charsetStr); + } + catch (Exception ex) { + throw new JBakeException("Unknown character set: " + charsetStr + " Try " + StandardCharsets.UTF_8.name()); + } + } + + @Override + public boolean getMakeImagesUrlAbolute() + { + return getAsBoolean(JBakeProperty.IMG_URL_MAKE_ABSOLUTE); + } + + @Override + public boolean getRelativeImagePathsPointToAssets() + { + return getAsBoolean(JBakeProperty.IMG_PATH_RELATIVE_POINTS_TO_ASSETS); + } + + + public void setDestinationFolderName(String folderName) { setProperty(JBakeProperty.DESTINATION_FOLDER, folderName); setupDefaultDestination(); @@ -544,4 +610,5 @@ public boolean getImgPathUpdate() { public void setImgPathUPdate(boolean imgPathUpdate) { setProperty(JBakeProperty.IMG_PATH_UPDATE, imgPathUpdate); } + } diff --git a/jbake-core/src/main/java/org/jbake/app/configuration/JBakeConfiguration.java b/jbake-core/src/main/java/org/jbake/app/configuration/JBakeConfiguration.java index 5e5435382..ec637e5fc 100644 --- a/jbake-core/src/main/java/org/jbake/app/configuration/JBakeConfiguration.java +++ b/jbake-core/src/main/java/org/jbake/app/configuration/JBakeConfiguration.java @@ -1,6 +1,7 @@ package org.jbake.app.configuration; import java.io.File; +import java.nio.charset.Charset; import java.util.Iterator; import java.util.List; @@ -299,6 +300,49 @@ public interface JBakeConfiguration { */ String getVersion(); + + /** + * Should JBake extract the title from the document? + */ + boolean getExtractTitleFromDoc(); + + /** + * Should JBake normalize HTML documents? true | false + */ + boolean getNormalizeHtml(); + + /** + * Should JBake convert HTML documents to XHTML? true | false + */ + boolean getConvertHtmlToXhtml(); + + /** + * Should JBake (and the Markup converters) pretty-print HTML output (if supported)? true | false + */ + boolean getPrettyPrintHtml(); + + /** + * The charset in which the resulting HTML files should be encoded. + */ + Charset getOutputHtmlCharset(); + + /** + * The charset in which the source files are encoded. + */ + Charset getInputCharset(); + + /** + * Should JBake prefix with site base URL? + */ + boolean getMakeImagesUrlAbolute(); + + /** + * Should JBake prefix with site base URL relative URLs? + * This is not disjunctive from IMAGES_URL_MAKE_ABSOLUTE. + */ + boolean getRelativeImagePathsPointToAssets(); + + /** * Set a property value for the given key * diff --git a/jbake-core/src/main/java/org/jbake/app/configuration/JBakeProperty.java b/jbake-core/src/main/java/org/jbake/app/configuration/JBakeProperty.java index 375d8f903..589f53b2e 100644 --- a/jbake-core/src/main/java/org/jbake/app/configuration/JBakeProperty.java +++ b/jbake-core/src/main/java/org/jbake/app/configuration/JBakeProperty.java @@ -45,8 +45,18 @@ public class JBakeProperty { public static final String URI_NO_EXTENSION_PREFIX = "uri.noExtension.prefix"; public static final String IMG_PATH_UPDATE = "img.path.update"; public static final String IMG_PATH_PREPEND_HOST = "img.path.prepend.host"; + public static final String IMG_URL_MAKE_ABSOLUTE = "img.url.makeAbsolute"; + public static final String IMG_PATH_RELATIVE_POINTS_TO_ASSETS = "img.path.relativePointsToAssets"; + public static final String VERSION = "version"; + public static final String EXTRACT_TITLE_FROM_DOC = "extract.title"; + public static final String HTML_NORMALIZE = "output.html.normalize"; + public static final String HTML_CONVERT_TO_XHTML = "output.html.convertToXhtml"; + public static final String HTML_PRETTY_PRINT = "output.html.prettyPrint"; + public static final String HTML_OUTPUT_CHARSET = "output.html.charset"; + public static final String INPUT_CHARSET = "input.charset"; + private JBakeProperty() {} } diff --git a/jbake-core/src/main/java/org/jbake/parser/AsciidoctorEngine.java b/jbake-core/src/main/java/org/jbake/parser/AsciidoctorEngine.java index 0abd2fa7f..188ea240b 100644 --- a/jbake-core/src/main/java/org/jbake/parser/AsciidoctorEngine.java +++ b/jbake-core/src/main/java/org/jbake/parser/AsciidoctorEngine.java @@ -83,7 +83,7 @@ private Asciidoctor getEngine(Options options) { } @Override - public void processHeader(final ParserContext context) { + public void parseHeaderBlock(final ParserContext context) { Options options = getAsciiDocOptionsAndAttributes(context); final Asciidoctor asciidoctor = getEngine(options); DocumentHeader header = asciidoctor.readDocumentHeader(context.getFile()); diff --git a/jbake-core/src/main/java/org/jbake/parser/ErrorEngine.java b/jbake-core/src/main/java/org/jbake/parser/ErrorEngine.java index bf654f6c8..2e2cdb068 100644 --- a/jbake-core/src/main/java/org/jbake/parser/ErrorEngine.java +++ b/jbake-core/src/main/java/org/jbake/parser/ErrorEngine.java @@ -5,6 +5,8 @@ import java.util.Date; import java.util.Map; +import org.jbake.app.Crawler.Attributes; + /** * An internal rendering engine used to notify the user that the markup format he used requires an engine that couldn't * be loaded. @@ -23,7 +25,7 @@ public ErrorEngine(final String name) { } @Override - public void processHeader(final ParserContext context) { + public void parseHeaderBlock(final ParserContext context) { Map contents = context.getDocumentModel(); contents.put(Attributes.TYPE, "post"); contents.put(Attributes.STATUS, "published"); diff --git a/jbake-core/src/main/java/org/jbake/parser/MarkupEngine.java b/jbake-core/src/main/java/org/jbake/parser/MarkupEngine.java index b0a4017df..8bfa268bb 100644 --- a/jbake-core/src/main/java/org/jbake/parser/MarkupEngine.java +++ b/jbake-core/src/main/java/org/jbake/parser/MarkupEngine.java @@ -1,15 +1,5 @@ package org.jbake.parser; -import org.apache.commons.configuration.CompositeConfiguration; -import org.apache.commons.configuration.Configuration; -import org.apache.commons.io.IOUtils; -import org.jbake.app.Crawler; -import org.jbake.app.configuration.DefaultJBakeConfiguration; -import org.jbake.app.configuration.JBakeConfiguration; -import org.json.simple.JSONValue; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -17,9 +7,24 @@ import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; +import org.apache.commons.configuration.CompositeConfiguration; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.jbake.app.Crawler; +import org.jbake.app.DebugUtil; +import org.jbake.app.JBakeException; +import org.jbake.app.configuration.DefaultJBakeConfiguration; +import org.jbake.app.configuration.JBakeConfiguration; +import org.json.simple.JSONValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Base class for markup engine wrappers. A markup engine is responsible for rendering @@ -35,6 +40,10 @@ public abstract class MarkupEngine implements ParserEngine { private JBakeConfiguration configuration; + public static final int MAX_HEADER_LINES = 50; + public static final Pattern HEADER_LINE_REGEX = Pattern.compile("^[\\p{Alnum}.-_]+\\p{Blank}*(=|:)\\p{Blank}*[^\\n\\p{Cntrl}]*\\p{Blank}*"); + + /** * Tests if this markup engine can process the document. * @@ -45,14 +54,33 @@ public boolean validate(ParserContext context) { return true; } + /** + * Validates the core requirements the documents have to fulfil. + */ + protected void validateInternal(ParserContext context) { + validateDocumentAttribute(context, Crawler.Attributes.TITLE, String.class); + validateDocumentAttribute(context, Crawler.Attributes.DATE, Date.class); + validateDocumentAttribute(context, Crawler.Attributes.TYPE, String.class); + } + + private void validateDocumentAttribute(ParserContext context, String attrName, Class expectedType) + { + Object type = context.getDocumentModel().get(attrName); + if (null == type) + throw new JBakeException("Document doesn't have a " + attrName + ": " + context.getFile().getPath()); + if (!expectedType.isAssignableFrom(type.getClass())) + throw new JBakeException("Document's " + attrName + " is not a " + expectedType.getName() + ", but " + type.getClass().getCanonicalName() + ": " + context.getFile().getPath()); + } + + /** * Processes the document header. Usually subclasses will parse the document body and look for * specific header metadata and export it into {@link ParserContext#getDocumentModel() contents} map. * * @param context the parser context */ - public void processHeader(final ParserContext context) { - } + /*public void parseHeaderBlock(final ParserContext context) { + }*/ /** * Processes the body of the document. Usually subclasses will parse the document body and render @@ -76,30 +104,29 @@ public Map parse(Configuration config, File file, String content */ public Map parse(JBakeConfiguration config, File file) { this.configuration = config; - List fileContents; - try (InputStream is = new FileInputStream(file)) { + List fileLines; - fileContents = IOUtils.readLines(is, config.getRenderEncoding()); + try (InputStream is = new FileInputStream(file)) { + // TODO: This should be done using streams for performance and memory reasons. + fileLines = IOUtils.readLines(is, config.getInputCharset()); } catch (IOException e) { LOGGER.error("Error while opening file {}", file, e); return null; } - boolean hasHeader = hasHeader(fileContents); + boolean hasHeader = hasHeader(fileLines); + ParserContext context = new ParserContext( file, - fileContents, + fileLines, config, - hasHeader - ); + hasHeader); - if (hasHeader) { - // read header from file - processDefaultHeader(context); - } - // then read engine specific headers - processHeader(context); + Map headersMap = parseHeaderBlock(context); + DebugUtil.printMap(headersMap, System.out); + + applyHeadersToDocument(headersMap, context.getDocumentModel()); setModelDefaultsIfNotSetInHeader(context); sanitizeTags(context); @@ -120,6 +147,9 @@ public Map parse(JBakeConfiguration config, File file) { LOGGER.error("Incomplete source file ({}) for markup engine: {}", file, getClass().getSimpleName()); return null; } + + validateInternal(context); + // TODO: post parsing plugins to hook in here? return context.getDocumentModel(); @@ -159,135 +189,80 @@ private void setModelDefaultsIfNotSetInHeader(ParserContext context) { /** * Checks if the file has a meta-data header. * - * @param contents Contents of file + * @param fileLines Contents of file * @return true if header exists, false if not */ - private boolean hasHeader(List contents) { - boolean headerValid = true; + private boolean hasHeader(List fileLines) + { + boolean headerSeparatorFound = false; boolean statusFound = false; boolean typeFound = false; - if (!headerSeparatorDemarcatesHeader(contents)) { - return false; - } + for (int i = 0; i < fileLines.size() && i < MAX_HEADER_LINES; i++) { + String line = fileLines.get(i); - for (String line : contents) { - if (hasHeaderSeparator(line)) { - LOGGER.debug("Header separator found"); + if (line.equals(configuration.getHeaderSeparator())) { + headerSeparatorFound = true; break; } - if (isTypeProperty(line)) { - LOGGER.debug("Type property found"); - typeFound = true; - } - - if (isStatusProperty(line)) { - LOGGER.debug("Status property found"); - statusFound = true; - } - if (!line.isEmpty() && !line.contains("=")) { - LOGGER.error("Property found without assignment [{}]", line); - headerValid = false; - } } - return headerValid && (statusFound || hasDefaultStatus()) && (typeFound || hasDefaultType()); - } - private boolean hasDefaultType() { - return !configuration.getDefaultType().isEmpty(); + return headerSeparatorFound; } - private boolean hasDefaultStatus() { - return !configuration.getDefaultStatus().isEmpty(); - } - - private boolean hasHeaderSeparatorInContent(List contents) { - return contents.indexOf(configuration.getHeaderSeparator()) != -1; - } /** - * Checks if header separator demarcates end of metadata header + * Process the header of the file. * - * @param contents - * @return true if header separator resides at end of metadata header, false if not + * @param context the parser context */ - private boolean headerSeparatorDemarcatesHeader(List contents) { - List subContents = null; - int index = contents.indexOf(configuration.getHeaderSeparator()); - if (index != -1) { - // get every line above header separator - subContents = contents.subList(0, index); - - for (String line : subContents) { - // header should only contain empty lines or lines with '=' in - if (!line.contains("=") && !line.isEmpty()) { - return false; - } - } - return true; - } else { - return false; - } - } + private Map parseHeaderBlock(ParserContext context) + { + String headerSeparator = configuration.getHeaderSeparator(); - private boolean hasHeaderSeparator(String line) { - return line.equals(configuration.getHeaderSeparator()); - } + Map headers = new HashMap<>(); - private boolean isStatusProperty(String line) { - return line.startsWith("status="); - } + List fileLines = context.getFileLines(); - private boolean isTypeProperty(String line) { - return line.startsWith("type="); - } + for (int i = 0; i < fileLines.size() && i < MAX_HEADER_LINES; i++) { + String line = fileLines.get(i); - /** - * Process the header of the file. - * - * @param context the parser context - */ - private void processDefaultHeader(ParserContext context) { - for (String line : context.getFileLines()) { - - if (hasHeaderSeparator(line)) { - break; + if (line.equals(headerSeparator)) { + return headers; } - processLine(line, context.getDocumentModel()); + + parsePotentialHeader(line, headers); } + // Only return headers if a separator line was found. + // Otherwise, prevent returning some randomly matching file content. + return Collections.emptyMap(); } - private void processLine(String line, Map content) { - String[] parts = line.split("=", 2); - if (!line.isEmpty() && parts.length == 2) { + private void parsePotentialHeader(String line, Map headersToFill) { + if (StringUtils.isBlank(line)) + return; + if (line.startsWith("#")) + return; - String key = sanitizeKey(parts[0]); - String value = sanitizeValue(parts[1]); + if (!HEADER_LINE_REGEX.matcher(line).matches()) + return; - if (key.equalsIgnoreCase(Crawler.Attributes.DATE)) { - DateFormat df = new SimpleDateFormat(configuration.getDateFormat()); - try { - Date date = df.parse(value); - content.put(key, date); - } catch (ParseException e) { - LOGGER.error("unable to parse date {}", value); - } - } else if (key.equalsIgnoreCase(Crawler.Attributes.TAGS)) { - content.put(key, getTags(value)); - } else if (isJson(value)) { - content.put(key, JSONValue.parse(value)); - } else { - content.put(key, value); - } - } + String[] parts = StringUtils.split(line, "=:", 2); + if (parts.length != 2) + return; + + String key = sanitizeKey(parts[0]); + String value = sanitizeValue(parts[1]); + + headersToFill.put(key, value); } - private String sanitizeValue(String part) { + private static String sanitizeValue(String part) { return part.trim(); } - private String sanitizeKey(String part) { + private static String sanitizeKey(String part) { String key; if (part.contains(UTF_8_BOM)) { key = part.trim().replace(UTF_8_BOM, ""); @@ -297,17 +272,50 @@ private String sanitizeKey(String part) { return key; } - private String[] getTags(String tagsPart) { + + private void applyHeadersToDocument(Map headers, Map documentModel) { + + for (Map.Entry header : headers.entrySet()) { + String key = header.getKey(); + String value = header.getValue(); + + // Convert date to Date + if (key.equalsIgnoreCase(Crawler.Attributes.DATE)) { + DateFormat df = new SimpleDateFormat(configuration.getDateFormat()); + try { + Date date = df.parse(value); + documentModel.put(key, date); + } catch (ParseException e) { + LOGGER.error("Unable to parse date: {}", value); + } + } + // Tags + else if (key.equalsIgnoreCase(Crawler.Attributes.TAGS)) { + documentModel.put(key, parseTags(value)); + } + // JSON + else if (isJson(value)) { + documentModel.put(key, JSONValue.parse(value)); + } + // Ordinary String + else { + documentModel.put(key, value); + } + } + } + + private static String[] parseTags(String tagsPart) { String[] tags = tagsPart.split(","); for (int i = 0; i < tags.length; i++) tags[i] = sanitizeValue(tags[i]); return tags; } - private boolean isJson(String part) { + private static boolean isJson(String part) { return part.startsWith("{") && part.endsWith("}"); } + /** * Process the body of the file. * @@ -332,4 +340,4 @@ private void processDefaultBody(ParserContext context) { } context.setBody(body.toString()); } -} \ No newline at end of file +} diff --git a/jbake-core/src/main/java/org/jbake/parser/RawMarkupEngine.java b/jbake-core/src/main/java/org/jbake/parser/RawMarkupEngine.java index 699d3bc9a..59320ad0d 100644 --- a/jbake-core/src/main/java/org/jbake/parser/RawMarkupEngine.java +++ b/jbake-core/src/main/java/org/jbake/parser/RawMarkupEngine.java @@ -1,4 +1,136 @@ package org.jbake.parser; +import java.io.ByteArrayInputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import org.apache.commons.lang.StringUtils; +import org.jbake.app.ConfigUtil; +import org.jbake.app.Crawler; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.nodes.Entities; +import org.jsoup.select.Elements; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * HTML markup engine. Processes .html files. + * TODO: Rename to HtmlMarkupEngine + */ public class RawMarkupEngine extends MarkupEngine { + + private static final Logger LOG = LoggerFactory.getLogger(RawMarkupEngine.class); + + /** + * Headers used as automatically detected titles will be marked with this class. + */ + public static final String CSS_CLASS_EXTRACTED_TITLE = "jbakeExtractedTitle"; + + @Override + public void processBody(ParserContext context) + { + super.processBody(context); + + boolean extractTitle = context.getConfig().getBoolean(ConfigUtil.Keys.EXTRACT_TITLE_FROM_DOC, true); + + Object currentTitle = context.getContents().get(Crawler.Attributes.TITLE); + extractTitle &= null == currentTitle || StringUtils.isBlank("" + currentTitle); + + + // Keeping everything off by default for backward compatibility. + boolean normalizeHtml = getFlag(context, ConfigUtil.Keys.HTML_NORMALIZE, false); + boolean convertToXhtml = getFlag(context, ConfigUtil.Keys.HTML_CONVERT_TO_XHTML, false); + boolean prettyPrint = getFlag(context, ConfigUtil.Keys.HTML_PRETTY_PRINT, false); + + // The input charset is already handled by MarkupEngine when it reads the file. + Charset inputCharset = StandardCharsets.UTF_8; //getCharset(context, ConfigUtil.Keys.HTML_INPUT_CHARSET, null); + Charset outputCharset = getCharset(context, ConfigUtil.Keys.HTML_OUTPUT_CHARSET, StandardCharsets.UTF_8); + + + if (extractTitle || normalizeHtml) { + + Document doc; + if (inputCharset == null) + doc = Jsoup.parse(context.getBody()); + else try { + doc = Jsoup.parse(new ByteArrayInputStream(context.getBody().getBytes()), inputCharset.name(), ""); + } catch (Exception ex) { + LOG.warn("Couldn't read a string?"); + doc = Jsoup.parse(context.getBody()); + } + + boolean domNeedsToBeSaved = false; + + findTitle: + if (extractTitle) { + + if (!StringUtils.isBlank(doc.title())) { + context.getContents().put(Crawler.Attributes.TITLE, doc.title()); + break findTitle; + } + + Elements headings = doc.select("h1, h2, h3, h4, h5, h6"); + if (headings.size() > 0) { + context.getContents().put(Crawler.Attributes.TITLE, headings.first().text()); + headings.first().addClass(CSS_CLASS_EXTRACTED_TITLE); + domNeedsToBeSaved = true; + break findTitle; + } + } + + if (doc.body() != null || normalizeHtml || convertToXhtml || domNeedsToBeSaved) { + Document.OutputSettings outputSettings = doc.outputSettings(); + + outputSettings.prettyPrint(prettyPrint); + if (outputCharset != null) + outputSettings.charset(outputCharset); + + if (convertToXhtml) { + outputSettings.escapeMode(Entities.EscapeMode.xhtml); + outputSettings.syntax(Document.OutputSettings.Syntax.xml); + } + else { + outputSettings.escapeMode(Entities.EscapeMode.extended); + outputSettings.syntax(Document.OutputSettings.Syntax.html); + } + + doc.outputSettings(outputSettings); + + Element elementToExport = doc.body(); + if (null == elementToExport) + elementToExport = doc; + context.getContents().put(Crawler.Attributes.BODY, elementToExport.html()); + context.getContents().put("charset", outputSettings.charset().name()); + } + } + } + + /** + * Handle invalid or unavailable charset. + */ + private Charset getCharset(ParserContext context, String configKey, Charset defaultCharset) + { + String charsetName = context.getConfig().getString(configKey); + try { + return Charset.forName(charsetName); + } + catch (Exception ex) { + LOG.warn("Invalid charset '{}' in '{}', resorting to '{}'", charsetName, ConfigUtil.Keys.HTML_OUTPUT_CHARSET, defaultCharset); + return defaultCharset; + } + } + + /** + * TODO: This should be in some ConfigUtils and happen when reading the config. + */ + private boolean getFlag(ParserContext context, String configKey, boolean defaultValue) { + try { + return context.getConfig().getBoolean(configKey, defaultValue); + } + catch (Exception ex) { + LOG.warn("Invalid configuration value for '{}', should be 'true' or 'false': {}", configKey, context.getConfig().getString(configKey)); + return defaultValue; + } + } } diff --git a/jbake-core/src/main/java/org/jbake/parser/texy/TexyRestService.java b/jbake-core/src/main/java/org/jbake/parser/texy/TexyRestService.java new file mode 100644 index 000000000..6ebc661f9 --- /dev/null +++ b/jbake-core/src/main/java/org/jbake/parser/texy/TexyRestService.java @@ -0,0 +1,50 @@ +package org.jbake.parser.texy; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import org.apache.commons.io.IOUtils; + +/** + * Texy converter that data and calls the remote Texy converting REST service. + * TODO: Could have a connection pool and could be parallelized. + */ +public class TexyRestService +{ + private final URL url; + + public TexyRestService(URL url) + { + this.url = url; + } + + /** + * Sends the given inputstream as a POST request to the configured Texy converted web service + * and returns an InputStream of the response body, which is the resulting XHTML. + */ + public InputStream convertTexyToXhtml(InputStream texyMarkupStream) + { + try { + HttpURLConnection conn = (HttpURLConnection) this.url.openConnection(); + conn.setRequestMethod("POST"); + conn.setRequestProperty("Content-Type", "text/texy"); + //conn.setRequestProperty("Content-Length", ""+); + + conn.setDoInput(true); + conn.setDoOutput(true); + BufferedInputStream bis = new BufferedInputStream(texyMarkupStream); + + try (OutputStream requestBodyStream = conn.getOutputStream()) { + IOUtils.copy(texyMarkupStream, requestBodyStream); + } + + return conn.getInputStream(); + } + catch (IOException ex) { + throw new RuntimeException("Call to Texy conversion service failed: " + ex.getMessage(), ex); + } + } +} diff --git a/jbake-core/src/main/java/org/jbake/parser/texy/TexyServiceEngine.java b/jbake-core/src/main/java/org/jbake/parser/texy/TexyServiceEngine.java new file mode 100644 index 000000000..d28939429 --- /dev/null +++ b/jbake-core/src/main/java/org/jbake/parser/texy/TexyServiceEngine.java @@ -0,0 +1,69 @@ +package org.jbake.parser.texy; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import org.apache.commons.lang3.StringUtils; +import org.jbake.app.Crawler; +import org.jbake.parser.MarkupEngine; +import org.jbake.parser.ParserContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Renders documents in the Texy syntax. + * + * @author Ondřej Žižka + */ +public class TexyServiceEngine extends MarkupEngine +{ + + private static final Logger LOG = LoggerFactory.getLogger(TexyServiceEngine.class); + + + @Override + public void parseHeaderBlock(ParserContext context) + { + super.parseHeaderBlock(context); + } + + @Override + public void processBody(final ParserContext context) { + String documentBody = context.getBody(); + + try (InputStream stream = new ByteArrayInputStream(documentBody.getBytes(StandardCharsets.UTF_8))){ + TexyRestService texyService = new TexyRestService(new URL("http://localhost:8022/TexyService.rest.php")); + + String xhtmlString = null; + try (InputStream xhtmlIS = texyService.convertTexyToXhtml(stream)){ + java.util.Scanner s = new java.util.Scanner(xhtmlIS, StandardCharsets.UTF_8.name()).useDelimiter("\\A"); + xhtmlString = s.hasNext() ? s.next() : ""; + context.setBody(xhtmlString); + + if (!context.getContents().containsKey(Crawler.Attributes.TITLE)) { + try { + new TitleExtractor().tryExtractHighestHeader(context); + } + catch (Exception ex){ + LOG.warn("Could not extract title from '{}': {}\nConverted XHTML: \n{}", context.getFile().getName(), ex.getMessage(), xhtmlString); + } + if (StringUtils.isBlank((String) context.getContents().get(Crawler.Attributes.TITLE))) + context.getContents().put(Crawler.Attributes.TITLE, context.getFile().getName()); + } + } + catch (IOException ex) { + String msg = "Couldn't convert:'" + context.getContentPath() + "': " + ex.getMessage(); + throw new RuntimeException(msg, ex); + // TOOO: I am not sure how to handle errors in JBake. The exception stops the whole process. + //LOG.warn(msg, ex); + //LOG.debug("Document's XHTML: \n" + xhtmlString); + } + } + catch (IOException ex) { + throw new RuntimeException("Failed opening String stream: " + ex.getMessage(), ex); + } + } + +} diff --git a/jbake-core/src/main/java/org/jbake/parser/texy/TitleExtractor.java b/jbake-core/src/main/java/org/jbake/parser/texy/TitleExtractor.java new file mode 100644 index 000000000..c7ab34bb0 --- /dev/null +++ b/jbake-core/src/main/java/org/jbake/parser/texy/TitleExtractor.java @@ -0,0 +1,112 @@ +package org.jbake.parser.texy; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Enumeration; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathFactory; +import org.apache.commons.collections.iterators.IteratorEnumeration; +import org.jbake.app.Crawler; +import org.jbake.parser.ParserContext; +import org.jbake.parser.RawMarkupEngine; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + + +/** + * Extracts a title from a XHTML document. + */ +public class TitleExtractor +{ + + /** + * Extracts the text of the first highest heading from the given XHTML. + * TODO: Move this whole extracting somewhere up to JBake! + */ + public void tryExtractHighestHeader(ParserContext context) + { + String xhtmlString = context.getBody(); + if (xhtmlString == null || "".equals(xhtmlString)) + return; + + // The result from texy needs to be wrapped, because it has no root element. + ByteArrayInputStream divStart = new ByteArrayInputStream("

".getBytes(StandardCharsets.UTF_8)); + ByteArrayInputStream divEnd = new ByteArrayInputStream("
".getBytes(StandardCharsets.UTF_8)); + ByteArrayInputStream is = new ByteArrayInputStream(xhtmlString.getBytes(StandardCharsets.UTF_8)); + Enumeration streams = new IteratorEnumeration(Arrays.asList(new InputStream[]{divStart, is, divEnd}).iterator()); + + try (SequenceInputStream wrapped = new SequenceInputStream(streams);) { + DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = builderFactory.newDocumentBuilder(); + Document xmlDocument = builder.parse(wrapped); + XPath xPath = XPathFactory.newInstance().newXPath(); + + // Find the first of the highest heading. + for (int i = 1; i <= 6; i++) { + // We want it to ignore namespaces, otherwise we would have to maintain a list of all XHTML namespaces. + String xpath = "//*[local-name()='h" + i + "']"; + NodeList nodeList = (NodeList) xPath.compile(xpath).evaluate(xmlDocument, XPathConstants.NODESET); + if (0 == nodeList.getLength()) + continue; + Element titleElm = (Element) nodeList.item(0); + titleElm.setAttribute("class", titleElm.getAttribute("class") + " " + RawMarkupEngine.CSS_CLASS_EXTRACTED_TITLE); + context.getContents().put(Crawler.Attributes.TITLE, titleElm.getTextContent()); + context.setBody(innerXml(xmlDocument)); + return; + } + } + catch (Exception e) { + throw new RuntimeException("Failed extracting a title: " + e.getMessage()); + } + } + + /** + * Getting an inner XML is a bit complicated in JDK. + * TODO: All this just to get rid if the previously added
? Let's use JSoup I guess. + */ + public static String innerXml(Node docRootNode) + { + try { + StringWriter writer = new StringWriter(); + + TransformerFactory tf = TransformerFactory.newInstance(); + Transformer transformer = tf.newTransformer(); + transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); + //transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + transformer.setOutputProperty(OutputKeys.ENCODING, "utf-8"); + + // Find the root element (may not be the first node). + Node rootElement = docRootNode.getFirstChild(); + while (rootElement != null && rootElement.getNodeType() != Node.ELEMENT_NODE) + rootElement = rootElement.getNextSibling(); + if (rootElement == null) + throw new RuntimeException("No root element found in given document node."); + + // Serialize the child nodes. + NodeList childNodes = rootElement.getChildNodes(); + for (int i = 0; i < childNodes.getLength(); i++) { + transformer.transform(new DOMSource(childNodes.item(i)), new StreamResult(writer)); + } + return writer.toString(); + + } + catch (Exception e) { + throw new RuntimeException("Failed getting inner XML for given node. " + e.getMessage()); + } + } + +} diff --git a/jbake-core/src/main/java/org/jbake/render/ArchiveRenderer.java b/jbake-core/src/main/java/org/jbake/render/ArchiveRenderer.java index ba6eebcb3..59c667c2a 100644 --- a/jbake-core/src/main/java/org/jbake/render/ArchiveRenderer.java +++ b/jbake-core/src/main/java/org/jbake/render/ArchiveRenderer.java @@ -1,5 +1,6 @@ package org.jbake.render; +import java.io.File; import org.apache.commons.configuration.CompositeConfiguration; import org.jbake.app.ContentStore; import org.jbake.app.Renderer; @@ -10,7 +11,7 @@ import java.io.File; -public class ArchiveRenderer implements RenderingTool { +public class ArchiveRenderer extends BaseRenderingTool implements RenderingTool { @Override public int render(Renderer renderer, ContentStore db, JBakeConfiguration config) throws RenderingException { @@ -33,4 +34,4 @@ public int render(Renderer renderer, ContentStore db, File destination, File tem return render(renderer, db, configuration); } -} \ No newline at end of file +} diff --git a/jbake-core/src/main/java/org/jbake/render/BaseRenderingTool.java b/jbake-core/src/main/java/org/jbake/render/BaseRenderingTool.java new file mode 100644 index 000000000..7cae19ec5 --- /dev/null +++ b/jbake-core/src/main/java/org/jbake/render/BaseRenderingTool.java @@ -0,0 +1,10 @@ +package org.jbake.render; + +public abstract class BaseRenderingTool implements RenderingTool +{ + @Override + public boolean isRendersInPlace() + { + return false; // Most renderers put the result elsewhere. + } +} diff --git a/jbake-core/src/main/java/org/jbake/render/DocumentsRenderer.java b/jbake-core/src/main/java/org/jbake/render/DocumentsRenderer.java index 59fe93e94..40a954243 100644 --- a/jbake-core/src/main/java/org/jbake/render/DocumentsRenderer.java +++ b/jbake-core/src/main/java/org/jbake/render/DocumentsRenderer.java @@ -17,6 +17,12 @@ public class DocumentsRenderer implements RenderingTool { + + @Override + public int render(Renderer renderer, ContentStore db, File destination, File templatesPath, CompositeConfiguration config) throws RenderingException { + return render(renderer, db, null); + } + @Override public int render(Renderer renderer, ContentStore db, JBakeConfiguration config) throws RenderingException { int renderedCount = 0; @@ -71,22 +77,22 @@ public int render(Renderer renderer, ContentStore db, JBakeConfiguration config) } } - /** - * Creates a simple content model to use in individual post navigations. - * - * @param document - * @return - */ - private Map getContentForNav(Map document) { - Map navDocument = new HashMap<>(); - navDocument.put(Attributes.NO_EXTENSION_URI, document.get(Attributes.NO_EXTENSION_URI)); - navDocument.put(Attributes.URI, document.get(Attributes.URI)); - navDocument.put(Attributes.TITLE, document.get(Attributes.TITLE)); - return navDocument; - } - - @Override - public int render(Renderer renderer, ContentStore db, File destination, File templatesPath, CompositeConfiguration config) throws RenderingException { - return render(renderer, db, null); - } -} \ No newline at end of file + @Override + public boolean isRendersInPlace() + { + return true; + } + + /** + * Creates a simple content model to use in individual post navigations. + * @param document + * @return + */ + private Map getContentForNav(Map document) { + Map navDocument = new HashMap(); + navDocument.put(Attributes.NO_EXTENSION_URI, document.get(Attributes.NO_EXTENSION_URI)); + navDocument.put(Attributes.URI, document.get(Attributes.URI)); + navDocument.put(Attributes.TITLE, document.get(Attributes.TITLE)); + return navDocument; + } +} diff --git a/jbake-core/src/main/java/org/jbake/render/FeedRenderer.java b/jbake-core/src/main/java/org/jbake/render/FeedRenderer.java index da956df33..7bbe42935 100644 --- a/jbake-core/src/main/java/org/jbake/render/FeedRenderer.java +++ b/jbake-core/src/main/java/org/jbake/render/FeedRenderer.java @@ -1,5 +1,6 @@ package org.jbake.render; +import java.io.File; import org.apache.commons.configuration.CompositeConfiguration; import org.jbake.app.ContentStore; import org.jbake.app.Renderer; @@ -10,7 +11,7 @@ import java.io.File; -public class FeedRenderer implements RenderingTool { +public class FeedRenderer extends BaseRenderingTool implements RenderingTool { @Override public int render(Renderer renderer, ContentStore db, JBakeConfiguration config) throws RenderingException { @@ -27,10 +28,11 @@ public int render(Renderer renderer, ContentStore db, JBakeConfiguration config) } } + ///REBASE isn't this in BaseRenderingTool? @Override public int render(Renderer renderer, ContentStore db, File destination, File templatesPath, CompositeConfiguration config) throws RenderingException { JBakeConfiguration configuration = new JBakeConfigurationFactory().createDefaultJbakeConfiguration(templatesPath.getParentFile(), config); return render(renderer, db, configuration); } -} \ No newline at end of file +} diff --git a/jbake-core/src/main/java/org/jbake/render/IndexRenderer.java b/jbake-core/src/main/java/org/jbake/render/IndexRenderer.java index bbdb7e4ae..d6aef0c04 100644 --- a/jbake-core/src/main/java/org/jbake/render/IndexRenderer.java +++ b/jbake-core/src/main/java/org/jbake/render/IndexRenderer.java @@ -1,5 +1,6 @@ package org.jbake.render; +import java.io.File; import org.apache.commons.configuration.CompositeConfiguration; import org.jbake.app.ContentStore; import org.jbake.app.Renderer; @@ -7,9 +8,7 @@ import org.jbake.app.configuration.JBakeConfigurationFactory; import org.jbake.template.RenderingException; -import java.io.File; - -public class IndexRenderer implements RenderingTool { +public class IndexRenderer extends BaseRenderingTool implements RenderingTool { @Override public int render(Renderer renderer, ContentStore db, JBakeConfiguration config) throws RenderingException { @@ -37,4 +36,4 @@ public int render(Renderer renderer, ContentStore db, File destination, File tem JBakeConfiguration configuration = new JBakeConfigurationFactory().createDefaultJbakeConfiguration(templatesPath.getParentFile(), config); return render(renderer, db, configuration); } -} \ No newline at end of file +} diff --git a/jbake-core/src/main/java/org/jbake/render/RenderingTool.java b/jbake-core/src/main/java/org/jbake/render/RenderingTool.java index 013eaaf2e..b6518dc1f 100644 --- a/jbake-core/src/main/java/org/jbake/render/RenderingTool.java +++ b/jbake-core/src/main/java/org/jbake/render/RenderingTool.java @@ -1,13 +1,12 @@ package org.jbake.render; +import java.io.File; import org.apache.commons.configuration.CompositeConfiguration; import org.jbake.app.ContentStore; import org.jbake.app.Renderer; import org.jbake.app.configuration.JBakeConfiguration; import org.jbake.template.RenderingException; -import java.io.File; - public interface RenderingTool { @@ -17,4 +16,9 @@ public interface RenderingTool { //TODO: remove at 3.0.0 int render(Renderer renderer, ContentStore db, File destination, File templatesPath, CompositeConfiguration config) throws RenderingException; -} \ No newline at end of file + /** + * Does this renderer create a file that will be situated in the same directory as the source markup? + * Serves to keep the URLs intact for renderers that do. + */ + boolean isRendersInPlace(); +} diff --git a/jbake-core/src/main/java/org/jbake/render/SitemapRenderer.java b/jbake-core/src/main/java/org/jbake/render/SitemapRenderer.java index 71bb5756b..3828ee3ac 100644 --- a/jbake-core/src/main/java/org/jbake/render/SitemapRenderer.java +++ b/jbake-core/src/main/java/org/jbake/render/SitemapRenderer.java @@ -1,5 +1,6 @@ package org.jbake.render; +import java.io.File; import org.apache.commons.configuration.CompositeConfiguration; import org.jbake.app.ContentStore; import org.jbake.app.Renderer; @@ -7,10 +8,8 @@ import org.jbake.app.configuration.JBakeConfigurationFactory; import org.jbake.template.RenderingException; -import java.io.File; - -public class SitemapRenderer implements RenderingTool { +public class SitemapRenderer extends BaseRenderingTool implements RenderingTool { @Override public int render(Renderer renderer, ContentStore db, JBakeConfiguration config) throws RenderingException { @@ -33,4 +32,4 @@ public int render(Renderer renderer, ContentStore db, File destination, File tem return render(renderer, db, configuration); } -} \ No newline at end of file +} diff --git a/jbake-core/src/main/java/org/jbake/render/TagsRenderer.java b/jbake-core/src/main/java/org/jbake/render/TagsRenderer.java index 2bb1f55df..8b5c7202c 100644 --- a/jbake-core/src/main/java/org/jbake/render/TagsRenderer.java +++ b/jbake-core/src/main/java/org/jbake/render/TagsRenderer.java @@ -1,5 +1,6 @@ package org.jbake.render; +import java.io.File; import org.apache.commons.configuration.CompositeConfiguration; import org.jbake.app.ContentStore; import org.jbake.app.Renderer; @@ -7,10 +8,8 @@ import org.jbake.app.configuration.JBakeConfigurationFactory; import org.jbake.template.RenderingException; -import java.io.File; - -public class TagsRenderer implements RenderingTool { +public class TagsRenderer extends BaseRenderingTool implements RenderingTool { @Override public int render(Renderer renderer, ContentStore db, JBakeConfiguration config) throws RenderingException { @@ -32,4 +31,4 @@ public int render(Renderer renderer, ContentStore db, File destination, File tem return render(renderer, db, configuration); } -} \ No newline at end of file +} diff --git a/jbake-core/src/main/java/org/jbake/util/HtmlUtil.java b/jbake-core/src/main/java/org/jbake/util/HtmlUtil.java index cbafe6674..2d056d3a4 100644 --- a/jbake-core/src/main/java/org/jbake/util/HtmlUtil.java +++ b/jbake-core/src/main/java/org/jbake/util/HtmlUtil.java @@ -1,5 +1,7 @@ package org.jbake.util; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.jbake.app.Crawler.Attributes; import org.jbake.app.configuration.JBakeConfiguration; import org.jsoup.Jsoup; @@ -7,8 +9,6 @@ import org.jsoup.nodes.Element; import org.jsoup.select.Elements; -import java.util.Map; - /** * @author Manik Magar */ @@ -21,61 +21,79 @@ private HtmlUtil() { * Image paths are specified as w.r.t. assets folder. This function prefix site host to all img src except * the ones that starts with http://, https://. *

- * If image path starts with "./", i.e. relative to the source file, then it first replace that with output file directory and the add site host. + * If the image path is relative to the source file (doesn't start with "/"), + * then it first replace that with output file directory and the add site host. * * @param fileContents Map representing file contents * @param configuration Configuration object + * + * TODO: This is too complicated, will need a refactor again. */ - public static void fixImageSourceUrls(Map fileContents, JBakeConfiguration configuration) { - String htmlContent = fileContents.get(Attributes.BODY).toString(); - boolean prependSiteHost = configuration.getImgPathPrependHost(); + public static void fixImageSourceUrls(Map fileContents, JBakeConfiguration configuration) + { String siteHost = configuration.getSiteHost(); - String uri = getDocumentUri(fileContents); + siteHost = StringUtils.appendIfMissing(siteHost, "/"); + + boolean prependSiteHost = configuration.getImgPathPrependHost(); + boolean relativePointsToAssets = configuration.getRelativeImagePathsPointToAssets(); + String dirUri = getDocumentUri(fileContents); + + String htmlContent = fileContents.get(Attributes.BODY).toString(); Document document = Jsoup.parseBodyFragment(htmlContent); - Elements allImgs = document.getElementsByTag("img"); + Elements allImgs = document.getElementsByTag("img"); for (Element img : allImgs) { - transformImageSource(img, uri, siteHost, prependSiteHost); + transformImageSource(img, dirUri, siteHost, prependSiteHost, relativePointsToAssets); } - //Use body().html() to prevent adding from parsed fragment. + // Use body().html() to prevent adding from parsed fragment. fileContents.put(Attributes.BODY, document.body().html()); } private static String getDocumentUri(Map fileContents) { - String uri = fileContents.get(Attributes.URI).toString(); + String dirUri = fileContents.get(Attributes.URI).toString(); if (fileContents.get(Attributes.NO_EXTENSION_URI) != null) { - uri = fileContents.get(Attributes.NO_EXTENSION_URI).toString(); - uri = removeTrailingSlash(uri); + dirUri = fileContents.get(Attributes.NO_EXTENSION_URI).toString(); + dirUri = removeTrailingSlash(dirUri); } - if (uri.contains("/")) { - uri = removeFilename(uri); + if (dirUri.contains("/")) { + dirUri = removeFilename(dirUri); } - return uri; + return dirUri; } - private static void transformImageSource(Element img, String uri, String siteHost, boolean prependSiteHost) { - String source = img.attr("src"); + private static void transformImageSource(Element img, String dirUri, String siteHost, + boolean prependSiteHost, + boolean relativePointsToAssets) { + String srcUrl = img.attr("src"); + if (srcUrl.startsWith("http://") || srcUrl.startsWith("https://")) + return; + + if (isRelativeToSourceMarkup(srcUrl, relativePointsToAssets)) + { + // Image relative to current content is explicitly specified, lets add dir URI to it. + srcUrl = dirUri + srcUrl; + } - // Now add the root path - if (!source.startsWith("http://") && !source.startsWith("https://")) { + //source = StringUtils.removeStart(source, "/"); - if (isRelative(source)) { - source = uri + source.replaceFirst("\\./", ""); + if (prependSiteHost) { + if (!siteHost.endsWith("/") && isRelative(srcUrl)) { + siteHost = siteHost.concat("/"); } + // Now add the base URL. + srcUrl = siteHost + srcUrl; + } - if (prependSiteHost) { - if (!siteHost.endsWith("/") && isRelative(source)) { - siteHost = siteHost.concat("/"); - } - source = siteHost + source; - } + img.attr("src", srcUrl); + } - img.attr("src", source); - } + private static boolean isRelativeToSourceMarkup(String srcUrl, boolean relativePointsToAssets) + { + return (relativePointsToAssets ? srcUrl.startsWith("./") : !srcUrl.startsWith("/")); } private static String removeFilename(String uri) { @@ -93,4 +111,5 @@ private static String removeTrailingSlash(String uri) { private static boolean isRelative(String source) { return !source.startsWith("/"); } + } diff --git a/jbake-core/src/main/resources/META-INF/org.jbake.parser.MarkupEngines.properties b/jbake-core/src/main/resources/META-INF/org.jbake.parser.MarkupEngines.properties index de6dddcf4..95587964a 100644 --- a/jbake-core/src/main/resources/META-INF/org.jbake.parser.MarkupEngines.properties +++ b/jbake-core/src/main/resources/META-INF/org.jbake.parser.MarkupEngines.properties @@ -1,3 +1,4 @@ org.jbake.parser.RawMarkupEngine=html org.jbake.parser.AsciidoctorEngine=ad,adoc,asciidoc -org.jbake.parser.MarkdownEngine=md \ No newline at end of file +org.jbake.parser.MarkdownEngine=md +org.jbake.parser.texy.TexyServiceEngine=texy diff --git a/jbake-core/src/main/resources/default.properties b/jbake-core/src/main/resources/default.properties index a6a66fa28..249382995 100644 --- a/jbake-core/src/main/resources/default.properties +++ b/jbake-core/src/main/resources/default.properties @@ -113,7 +113,17 @@ index.posts_per_page=3 site.host=http://www.jbake.org # String used to separate the header from the body header.separator=~~~~~~ -# update image path + +# Update image path? img.path.update=false # Prepend site.host to image paths -img.path.prepend.host=true \ No newline at end of file +img.path.prepend.host=true +# Should JBake make URLs full? I.e. http://host/path +# If false, they remain relative, but will be adjusted when rendered outside the original location. +# That may be useful when the rendered HTML is deployed to different hosts or loaded from a filesystem. +img.url.makeAbsolute=true + +# Should JBake treat relative (except for "./") as relative to root? +# That means, will be looked up from the site root. +# If false, then they are relative to the file where they are used. +img.url.relativePointsToAssets=true diff --git a/jbake-dist/src/dist/lib/logging/logback.xml b/jbake-dist/src/dist/lib/logging/logback.xml index 22cd77f5a..73f33a255 100644 --- a/jbake-dist/src/dist/lib/logging/logback.xml +++ b/jbake-dist/src/dist/lib/logging/logback.xml @@ -14,6 +14,7 @@ + From fe97417ed621f9ba5c49d6a70182773d1dd9661a Mon Sep 17 00:00:00 2001 From: Ondrej Zizka Date: Tue, 30 Oct 2018 04:35:11 +0100 Subject: [PATCH 2/6] Fixes after grand rebase --- .../main/java/org/jbake/app/ConfigUtil.java | 0 .../src/main/java/org/jbake/app/Oven.java | 2 +- .../DefaultJBakeConfiguration.java | 29 ++++++++++-- .../org/jbake/parser/AsciidoctorEngine.java | 4 +- .../java/org/jbake/parser/ErrorEngine.java | 5 +-- .../java/org/jbake/parser/MarkupEngine.java | 2 +- .../org/jbake/parser/RawMarkupEngine.java | 44 +++++++------------ .../jbake/parser/texy/TexyServiceEngine.java | 13 +++--- .../org/jbake/parser/texy/TitleExtractor.java | 2 +- 9 files changed, 55 insertions(+), 46 deletions(-) create mode 100644 jbake-core/src/main/java/org/jbake/app/ConfigUtil.java diff --git a/jbake-core/src/main/java/org/jbake/app/ConfigUtil.java b/jbake-core/src/main/java/org/jbake/app/ConfigUtil.java new file mode 100644 index 000000000..e69de29bb diff --git a/jbake-core/src/main/java/org/jbake/app/Oven.java b/jbake-core/src/main/java/org/jbake/app/Oven.java index 7842545dc..c96614127 100644 --- a/jbake-core/src/main/java/org/jbake/app/Oven.java +++ b/jbake-core/src/main/java/org/jbake/app/Oven.java @@ -164,7 +164,7 @@ public void bake() { * Replaces the URLs to resources in documents so that they are pointing to the resources even if the rendered document is placed * somewhere else than the source markup. */ - private void makeUrlsAbsolute(ContentStore db, CompositeConfiguration config) + private void makeUrlsAbsolute(ContentStore db, JBakeConfiguration config) { int renderedCount = 0; final List errors = new LinkedList(); diff --git a/jbake-core/src/main/java/org/jbake/app/configuration/DefaultJBakeConfiguration.java b/jbake-core/src/main/java/org/jbake/app/configuration/DefaultJBakeConfiguration.java index 5c850ef54..76ca6161a 100644 --- a/jbake-core/src/main/java/org/jbake/app/configuration/DefaultJBakeConfiguration.java +++ b/jbake-core/src/main/java/org/jbake/app/configuration/DefaultJBakeConfiguration.java @@ -22,6 +22,7 @@ */ public class DefaultJBakeConfiguration implements JBakeConfiguration { + private static final Logger LOG = LoggerFactory.getLogger(DefaultJBakeConfiguration.class); private static final String SOURCE_FOLDER_KEY = "sourceFolder"; private static final String DESTINATION_FOLDER_KEY = "destinationFolder"; @@ -32,7 +33,7 @@ public class DefaultJBakeConfiguration implements JBakeConfiguration { private static final String DOCTYPE_FILE_POSTFIX = ".file"; private static final String DOCTYPE_EXTENSION_POSTFIX = ".extension"; private static final String DOCTYPE_TEMPLATE_PREFIX = "template."; - private Logger logger = LoggerFactory.getLogger(DefaultJBakeConfiguration.class); + private CompositeConfiguration compositeConfiguration; /** @@ -67,6 +68,10 @@ private boolean getAsBoolean(String key) { return compositeConfiguration.getBoolean(key, false); } + private boolean getAsBoolean(String key, boolean defaultValue) { + return compositeConfiguration.getBoolean(key, defaultValue); + } + private File getAsFolder(String key) { return (File) get(key); } @@ -87,6 +92,22 @@ private String getAsString(String key, String defaultValue) { return compositeConfiguration.getString(key, defaultValue); } + /** + * Handle invalid or unavailable charset. + */ + private Charset getAsCharset(String key, Charset defaultCharset) + { + String charsetName = compositeConfiguration.getString(key); + try { + return Charset.forName(charsetName); + } + catch (Exception ex) { + LOG.warn("Invalid charset '{}' in '{}', resorting to '{}'", charsetName, key, defaultCharset); + return defaultCharset; + } + } + + @Override public List getAsciidoctorAttributes() { return getAsList(JBakeProperty.ASCIIDOCTOR_ATTRIBUTES); @@ -97,7 +118,7 @@ public Object getAsciidoctorOption(String optionKey) { Object value = subConfig.getProperty(optionKey); if (value == null) { - logger.warn("Cannot find asciidoctor option '{}.{}'", JBakeProperty.ASCIIDOCTOR_OPTION, optionKey); + LOG.warn("Cannot find asciidoctor option '{}.{}'", JBakeProperty.ASCIIDOCTOR_OPTION, optionKey); return ""; } return value; @@ -431,7 +452,7 @@ public File getTemplateFileByDocType(String docType) { if (templateFileName != null) { return new File(getTemplateFolder(), templateFileName); } - logger.warn("Cannot find configuration key '{}' for document type '{}'", templateKey, docType); + LOG.warn("Cannot find configuration key '{}' for document type '{}'", templateKey, docType); return null; } @@ -475,7 +496,7 @@ public String getVersion() { @Override public boolean getExtractTitleFromDoc() { - return getAsBoolean(JBakeProperty.EXTRACT_TITLE_FROM_DOC); + return getAsBoolean(JBakeProperty.EXTRACT_TITLE_FROM_DOC, true); } @Override diff --git a/jbake-core/src/main/java/org/jbake/parser/AsciidoctorEngine.java b/jbake-core/src/main/java/org/jbake/parser/AsciidoctorEngine.java index 188ea240b..47152c123 100644 --- a/jbake-core/src/main/java/org/jbake/parser/AsciidoctorEngine.java +++ b/jbake-core/src/main/java/org/jbake/parser/AsciidoctorEngine.java @@ -83,7 +83,7 @@ private Asciidoctor getEngine(Options options) { } @Override - public void parseHeaderBlock(final ParserContext context) { + public Map parseHeaderBlock(final ParserContext context) { Options options = getAsciiDocOptionsAndAttributes(context); final Asciidoctor asciidoctor = getEngine(options); DocumentHeader header = asciidoctor.readDocumentHeader(context.getFile()); @@ -121,6 +121,8 @@ public void parseHeaderBlock(final ParserContext context) { documentModel.put(key, attributes.get(key)); } } + + return null; // TODO: Create the header map first, then apply to the doc. } private boolean canCastToString(Object value) { diff --git a/jbake-core/src/main/java/org/jbake/parser/ErrorEngine.java b/jbake-core/src/main/java/org/jbake/parser/ErrorEngine.java index 2e2cdb068..88ba0fe82 100644 --- a/jbake-core/src/main/java/org/jbake/parser/ErrorEngine.java +++ b/jbake-core/src/main/java/org/jbake/parser/ErrorEngine.java @@ -5,8 +5,6 @@ import java.util.Date; import java.util.Map; -import org.jbake.app.Crawler.Attributes; - /** * An internal rendering engine used to notify the user that the markup format he used requires an engine that couldn't * be loaded. @@ -25,7 +23,7 @@ public ErrorEngine(final String name) { } @Override - public void parseHeaderBlock(final ParserContext context) { + public Map parseHeaderBlock(final ParserContext context) { Map contents = context.getDocumentModel(); contents.put(Attributes.TYPE, "post"); contents.put(Attributes.STATUS, "published"); @@ -33,6 +31,7 @@ public void parseHeaderBlock(final ParserContext context) { contents.put(Attributes.DATE, new Date()); contents.put(Attributes.TAGS, new String[0]); contents.put(Attributes.ID, context.getFile().getName()); + return null; // TODO: Create the header map first, then apply to the doc. } @Override diff --git a/jbake-core/src/main/java/org/jbake/parser/MarkupEngine.java b/jbake-core/src/main/java/org/jbake/parser/MarkupEngine.java index 8bfa268bb..725d60a63 100644 --- a/jbake-core/src/main/java/org/jbake/parser/MarkupEngine.java +++ b/jbake-core/src/main/java/org/jbake/parser/MarkupEngine.java @@ -216,7 +216,7 @@ private boolean hasHeader(List fileLines) * * @param context the parser context */ - private Map parseHeaderBlock(ParserContext context) + public Map parseHeaderBlock(ParserContext context) { String headerSeparator = configuration.getHeaderSeparator(); diff --git a/jbake-core/src/main/java/org/jbake/parser/RawMarkupEngine.java b/jbake-core/src/main/java/org/jbake/parser/RawMarkupEngine.java index 59320ad0d..371796031 100644 --- a/jbake-core/src/main/java/org/jbake/parser/RawMarkupEngine.java +++ b/jbake-core/src/main/java/org/jbake/parser/RawMarkupEngine.java @@ -4,7 +4,6 @@ import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import org.apache.commons.lang.StringUtils; -import org.jbake.app.ConfigUtil; import org.jbake.app.Crawler; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -32,20 +31,20 @@ public void processBody(ParserContext context) { super.processBody(context); - boolean extractTitle = context.getConfig().getBoolean(ConfigUtil.Keys.EXTRACT_TITLE_FROM_DOC, true); + boolean extractTitle = context.getConfig().getExtractTitleFromDoc(); - Object currentTitle = context.getContents().get(Crawler.Attributes.TITLE); + Object currentTitle = context.getDocumentModel().get(Crawler.Attributes.TITLE); extractTitle &= null == currentTitle || StringUtils.isBlank("" + currentTitle); // Keeping everything off by default for backward compatibility. - boolean normalizeHtml = getFlag(context, ConfigUtil.Keys.HTML_NORMALIZE, false); - boolean convertToXhtml = getFlag(context, ConfigUtil.Keys.HTML_CONVERT_TO_XHTML, false); - boolean prettyPrint = getFlag(context, ConfigUtil.Keys.HTML_PRETTY_PRINT, false); + boolean normalizeHtml = context.getConfig().getNormalizeHtml(); // Default false + boolean convertToXhtml = context.getConfig().getConvertHtmlToXhtml(); + boolean prettyPrint = context.getConfig().getPrettyPrintHtml(); // The input charset is already handled by MarkupEngine when it reads the file. - Charset inputCharset = StandardCharsets.UTF_8; //getCharset(context, ConfigUtil.Keys.HTML_INPUT_CHARSET, null); - Charset outputCharset = getCharset(context, ConfigUtil.Keys.HTML_OUTPUT_CHARSET, StandardCharsets.UTF_8); + Charset inputCharset = StandardCharsets.UTF_8; // context.getConfig().getInputCharset(); + Charset outputCharset = context.getConfig().getOutputHtmlCharset(); // Default: UTF-8 if (extractTitle || normalizeHtml) { @@ -66,13 +65,13 @@ public void processBody(ParserContext context) if (extractTitle) { if (!StringUtils.isBlank(doc.title())) { - context.getContents().put(Crawler.Attributes.TITLE, doc.title()); + context.getDocumentModel().put(Crawler.Attributes.TITLE, doc.title()); break findTitle; } Elements headings = doc.select("h1, h2, h3, h4, h5, h6"); if (headings.size() > 0) { - context.getContents().put(Crawler.Attributes.TITLE, headings.first().text()); + context.getDocumentModel().put(Crawler.Attributes.TITLE, headings.first().text()); headings.first().addClass(CSS_CLASS_EXTRACTED_TITLE); domNeedsToBeSaved = true; break findTitle; @@ -100,37 +99,24 @@ public void processBody(ParserContext context) Element elementToExport = doc.body(); if (null == elementToExport) elementToExport = doc; - context.getContents().put(Crawler.Attributes.BODY, elementToExport.html()); - context.getContents().put("charset", outputSettings.charset().name()); + context.getDocumentModel().put(Crawler.Attributes.BODY, elementToExport.html()); + context.getDocumentModel().put("charset", outputSettings.charset().name()); } } } - /** - * Handle invalid or unavailable charset. - */ - private Charset getCharset(ParserContext context, String configKey, Charset defaultCharset) - { - String charsetName = context.getConfig().getString(configKey); - try { - return Charset.forName(charsetName); - } - catch (Exception ex) { - LOG.warn("Invalid charset '{}' in '{}', resorting to '{}'", charsetName, ConfigUtil.Keys.HTML_OUTPUT_CHARSET, defaultCharset); - return defaultCharset; - } - } + /** * TODO: This should be in some ConfigUtils and happen when reading the config. - */ + * private boolean getFlag(ParserContext context, String configKey, boolean defaultValue) { try { return context.getConfig().getBoolean(configKey, defaultValue); } catch (Exception ex) { - LOG.warn("Invalid configuration value for '{}', should be 'true' or 'false': {}", configKey, context.getConfig().getString(configKey)); + LOG.warn("Invalid configuration value for '{}', should be 'true' or 'false': {}", configKey, ... ); return defaultValue; } - } + }/**/ } diff --git a/jbake-core/src/main/java/org/jbake/parser/texy/TexyServiceEngine.java b/jbake-core/src/main/java/org/jbake/parser/texy/TexyServiceEngine.java index d28939429..d11e79051 100644 --- a/jbake-core/src/main/java/org/jbake/parser/texy/TexyServiceEngine.java +++ b/jbake-core/src/main/java/org/jbake/parser/texy/TexyServiceEngine.java @@ -5,6 +5,7 @@ import java.io.InputStream; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.jbake.app.Crawler; import org.jbake.parser.MarkupEngine; @@ -24,9 +25,9 @@ public class TexyServiceEngine extends MarkupEngine @Override - public void parseHeaderBlock(ParserContext context) + public Map parseHeaderBlock(ParserContext context) { - super.parseHeaderBlock(context); + return super.parseHeaderBlock(context); } @Override @@ -42,19 +43,19 @@ public void processBody(final ParserContext context) { xhtmlString = s.hasNext() ? s.next() : ""; context.setBody(xhtmlString); - if (!context.getContents().containsKey(Crawler.Attributes.TITLE)) { + if (!context.getDocumentModel().containsKey(Crawler.Attributes.TITLE)) { try { new TitleExtractor().tryExtractHighestHeader(context); } catch (Exception ex){ LOG.warn("Could not extract title from '{}': {}\nConverted XHTML: \n{}", context.getFile().getName(), ex.getMessage(), xhtmlString); } - if (StringUtils.isBlank((String) context.getContents().get(Crawler.Attributes.TITLE))) - context.getContents().put(Crawler.Attributes.TITLE, context.getFile().getName()); + if (StringUtils.isBlank((String) context.getDocumentModel().get(Crawler.Attributes.TITLE))) + context.getDocumentModel().put(Crawler.Attributes.TITLE, context.getFile().getName()); } } catch (IOException ex) { - String msg = "Couldn't convert:'" + context.getContentPath() + "': " + ex.getMessage(); + String msg = "Couldn't convert:'" + context.getFile().getPath() + "': " + ex.getMessage(); throw new RuntimeException(msg, ex); // TOOO: I am not sure how to handle errors in JBake. The exception stops the whole process. //LOG.warn(msg, ex); diff --git a/jbake-core/src/main/java/org/jbake/parser/texy/TitleExtractor.java b/jbake-core/src/main/java/org/jbake/parser/texy/TitleExtractor.java index c7ab34bb0..cdb1703eb 100644 --- a/jbake-core/src/main/java/org/jbake/parser/texy/TitleExtractor.java +++ b/jbake-core/src/main/java/org/jbake/parser/texy/TitleExtractor.java @@ -64,7 +64,7 @@ public void tryExtractHighestHeader(ParserContext context) continue; Element titleElm = (Element) nodeList.item(0); titleElm.setAttribute("class", titleElm.getAttribute("class") + " " + RawMarkupEngine.CSS_CLASS_EXTRACTED_TITLE); - context.getContents().put(Crawler.Attributes.TITLE, titleElm.getTextContent()); + context.getDocumentModel().put(Crawler.Attributes.TITLE, titleElm.getTextContent()); context.setBody(innerXml(xmlDocument)); return; } From 5172c3a8412ad057f2839ae71638856e220c8e50 Mon Sep 17 00:00:00 2001 From: Ondrej Zizka Date: Tue, 30 Oct 2018 04:36:11 +0100 Subject: [PATCH 3/6] Add .../out/ to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 82ab65eb3..4179f3861 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ build/ .gradle .gradletasknamecache +jbake-core/out/ +jbake-dist/out/ From 9bbf00917a798440628b73def10ec9b2ef5e826e Mon Sep 17 00:00:00 2001 From: Ondrej Zizka Date: Tue, 30 Oct 2018 04:46:59 +0100 Subject: [PATCH 4/6] Make javadoc lint happy --- jbake-core/src/main/java/org/jbake/app/ContentStore.java | 2 +- .../java/org/jbake/app/configuration/JBakeConfiguration.java | 4 ++-- .../src/main/java/org/jbake/parser/AsciidoctorEngine.java | 2 +- .../src/main/java/org/jbake/parser/texy/TitleExtractor.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jbake-core/src/main/java/org/jbake/app/ContentStore.java b/jbake-core/src/main/java/org/jbake/app/ContentStore.java index 6ddae6523..0d4384c7b 100644 --- a/jbake-core/src/main/java/org/jbake/app/ContentStore.java +++ b/jbake-core/src/main/java/org/jbake/app/ContentStore.java @@ -176,7 +176,7 @@ private void activateOnCurrentThread() { /** * Get a document by sourceUri and update it from the given map. * @return the saved document. - * @throws Exception if sourceUri or docType are null, or if the document doesn't exist. + * throws {@link IllegalArgumentException} if sourceUri or docType are null, or if the document doesn't exist. */ public ODocument mergeDocument(Map incomingDocMap) { diff --git a/jbake-core/src/main/java/org/jbake/app/configuration/JBakeConfiguration.java b/jbake-core/src/main/java/org/jbake/app/configuration/JBakeConfiguration.java index ec637e5fc..95d5c23e1 100644 --- a/jbake-core/src/main/java/org/jbake/app/configuration/JBakeConfiguration.java +++ b/jbake-core/src/main/java/org/jbake/app/configuration/JBakeConfiguration.java @@ -332,12 +332,12 @@ public interface JBakeConfiguration { Charset getInputCharset(); /** - * Should JBake prefix with site base URL? + * Should JBake prefix <img src="..."> with site base URL? */ boolean getMakeImagesUrlAbolute(); /** - * Should JBake prefix with site base URL relative URLs? + * Should JBake prefix <img src="..."> with site base URL relative URLs? * This is not disjunctive from IMAGES_URL_MAKE_ABSOLUTE. */ boolean getRelativeImagePathsPointToAssets(); diff --git a/jbake-core/src/main/java/org/jbake/parser/AsciidoctorEngine.java b/jbake-core/src/main/java/org/jbake/parser/AsciidoctorEngine.java index 47152c123..b6ba589c3 100644 --- a/jbake-core/src/main/java/org/jbake/parser/AsciidoctorEngine.java +++ b/jbake-core/src/main/java/org/jbake/parser/AsciidoctorEngine.java @@ -121,7 +121,7 @@ public Map parseHeaderBlock(final ParserContext context) { documentModel.put(key, attributes.get(key)); } } - + return null; // TODO: Create the header map first, then apply to the doc. } diff --git a/jbake-core/src/main/java/org/jbake/parser/texy/TitleExtractor.java b/jbake-core/src/main/java/org/jbake/parser/texy/TitleExtractor.java index cdb1703eb..644b1616c 100644 --- a/jbake-core/src/main/java/org/jbake/parser/texy/TitleExtractor.java +++ b/jbake-core/src/main/java/org/jbake/parser/texy/TitleExtractor.java @@ -76,7 +76,7 @@ public void tryExtractHighestHeader(ParserContext context) /** * Getting an inner XML is a bit complicated in JDK. - * TODO: All this just to get rid if the previously added

? Let's use JSoup I guess. + * TODO: All this just to get rid if the previously added <div/>? Let's use JSoup I guess. */ public static String innerXml(Node docRootNode) { From c7a3b5d8e171e50d6e713d8e80932ad76197b64b Mon Sep 17 00:00:00 2001 From: Ondrej Zizka Date: Tue, 30 Oct 2018 05:00:34 +0100 Subject: [PATCH 5/6] Don't require output.html.charset, use UTF-8 --- .../jbake/app/configuration/DefaultJBakeConfiguration.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jbake-core/src/main/java/org/jbake/app/configuration/DefaultJBakeConfiguration.java b/jbake-core/src/main/java/org/jbake/app/configuration/DefaultJBakeConfiguration.java index 76ca6161a..3c9694442 100644 --- a/jbake-core/src/main/java/org/jbake/app/configuration/DefaultJBakeConfiguration.java +++ b/jbake-core/src/main/java/org/jbake/app/configuration/DefaultJBakeConfiguration.java @@ -525,7 +525,8 @@ public Charset getOutputHtmlCharset() return Charset.forName(charsetStr); } catch (Exception ex) { - throw new JBakeException("Unknown character set: " + charsetStr + " Try " + StandardCharsets.UTF_8.name()); + //throw new JBakeException("Unknown character set: " + charsetStr + " Try " + StandardCharsets.UTF_8.name()); + return StandardCharsets.UTF_8; } } @@ -537,7 +538,8 @@ public Charset getInputCharset() return Charset.forName(charsetStr); } catch (Exception ex) { - throw new JBakeException("Unknown character set: " + charsetStr + " Try " + StandardCharsets.UTF_8.name()); + //throw new JBakeException("Unknown character set: " + charsetStr + " Try " + StandardCharsets.UTF_8.name()); + return StandardCharsets.UTF_8; } } From b95f8d07500968de4d4ddcad0de75734635d40ab Mon Sep 17 00:00:00 2001 From: Ondrej Zizka Date: Tue, 30 Oct 2018 05:34:10 +0100 Subject: [PATCH 6/6] Fix bugs revealed by tests --- .../src/main/java/org/jbake/app/DebugUtil.java | 17 ++++++++++++++++- .../src/main/java/org/jbake/app/Oven.java | 10 ++++------ .../java/org/jbake/parser/MarkupEngine.java | 14 +++++++++++--- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/jbake-core/src/main/java/org/jbake/app/DebugUtil.java b/jbake-core/src/main/java/org/jbake/app/DebugUtil.java index 001169930..2ade26e70 100644 --- a/jbake-core/src/main/java/org/jbake/app/DebugUtil.java +++ b/jbake-core/src/main/java/org/jbake/app/DebugUtil.java @@ -1,11 +1,26 @@ package org.jbake.app; import java.io.PrintStream; +import java.sql.SQLOutput; import java.util.Map; +import org.apache.commons.lang.StringUtils; public class DebugUtil { - public static void printMap(Map map, PrintStream printStream){ + public static void printMap(String label, Map map, PrintStream printStream){ + if (null == map) { + printStream.println("The Map is null."); + return; + } + + if (null == printStream) { + printStream = System.out; + return; + } + if (!StringUtils.isBlank(label)) { + printStream.println(label + ":"); + } + printStream.println(); for (Map.Entry entry: map.entrySet()) { printStream.println(entry.getKey() + " :: " + entry.getValue()); diff --git a/jbake-core/src/main/java/org/jbake/app/Oven.java b/jbake-core/src/main/java/org/jbake/app/Oven.java index c96614127..14b29cf09 100644 --- a/jbake-core/src/main/java/org/jbake/app/Oven.java +++ b/jbake-core/src/main/java/org/jbake/app/Oven.java @@ -140,13 +140,11 @@ public void bake() { // render content renderContent(); - { - // copy assets - asset.copy(); - asset.copyAssetsFromContent(config.getContentFolder()); + // copy assets + asset.copy(); + asset.copyAssetsFromContent(config.getContentFolder()); - errors.addAll(asset.getErrors()); - } + errors.addAll(asset.getErrors()); LOGGER.info("Baking finished!"); long end = new Date().getTime(); diff --git a/jbake-core/src/main/java/org/jbake/parser/MarkupEngine.java b/jbake-core/src/main/java/org/jbake/parser/MarkupEngine.java index 725d60a63..f5594c99b 100644 --- a/jbake-core/src/main/java/org/jbake/parser/MarkupEngine.java +++ b/jbake-core/src/main/java/org/jbake/parser/MarkupEngine.java @@ -124,9 +124,10 @@ public Map parse(JBakeConfiguration config, File file) { hasHeader); Map headersMap = parseHeaderBlock(context); - DebugUtil.printMap(headersMap, System.out); + DebugUtil.printMap("Headers of " + file, headersMap, System.out); - applyHeadersToDocument(headersMap, context.getDocumentModel()); + if (headersMap != null) + applyHeadersToDocument(headersMap, context.getDocumentModel()); setModelDefaultsIfNotSetInHeader(context); sanitizeTags(context); @@ -212,7 +213,14 @@ private boolean hasHeader(List fileLines) /** - * Process the header of the file. + * Parse the headers of the file being parsed in given context. + * + * This method should parse the headers and return them for further processing. + * Otherwise the implementation may opt to apply the headers to the document itself and return null. + * + * TODO: The processing should be flatter in general, IMHO. + * Rather than nesting X levels deep to process a document, config overlays, an index, etc., + * the code blocks should pass packages of information around. Otherwise making JBake pluggable would be harder. * * @param context the parser context */