From ffeb5d6dbe62f78dfc2c66dca741107d195f1f98 Mon Sep 17 00:00:00 2001 From: "manik.magar" Date: Fri, 24 Jun 2016 21:19:23 -0400 Subject: [PATCH 1/3] Added Permalink feature --- src/main/java/org/jbake/app/Crawler.java | 83 ++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/src/main/java/org/jbake/app/Crawler.java b/src/main/java/org/jbake/app/Crawler.java index 1927d18c2..7977a07fd 100644 --- a/src/main/java/org/jbake/app/Crawler.java +++ b/src/main/java/org/jbake/app/Crawler.java @@ -11,15 +11,18 @@ import static java.io.File.separator; import java.io.File; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.time.DateFormatUtils; import com.orientechnologies.orient.core.record.impl.ODocument; @@ -178,6 +181,9 @@ private void crawlSourceFile(final File sourceFile, final String sha1, final Str } } + buildPermalink(fileContents); + + if (config.getBoolean(Keys.URI_NO_EXTENSION)) { fileContents.put(Attributes.NO_EXTENSION_URI, uri.replace("/index.html", "/")); } @@ -191,6 +197,83 @@ private void crawlSourceFile(final File sourceFile, final String sha1, final Str LOGGER.warn("{} has an invalid header, it has been ignored!", sourceFile); } } + + /** + * This function generates permalinks if they are enabled in configuration. + * + * Sample configuration entry: permalink.pattern = + * + * @param fileContents + * @author Manik Magar + * @return + */ + private String buildPermalink(Map fileContents){ + String permalink = (String) fileContents.get(Attributes.URI); + + boolean permalinkEnabled = config.getBoolean("permalink"); + if (permalinkEnabled) { + + String pattern = config.getString("permalink.pattern"); + + String[] parts = pattern.split("/:"); + List pLink = new ArrayList(); + for (String part : parts){ + part = part.trim().replace("/", ""); + if (part.endsWith(":")){ + pLink.add(part.replace(":", "")); + } else if(fileContents.containsKey(part)){ + Object value = fileContents.get(part); + if (value instanceof String){ + pLink.add(value.toString()); + } else if (value.getClass().equals(String[].class)){ + pLink.addAll(Arrays.asList((String[])value)); + } + } else if (Arrays.asList("YEAR","MONTH","DAY").contains(part)) { + Date publishedDate = (Date) fileContents.get("date"); + if(Objects.nonNull(publishedDate)){ + String dateValue = null; + if(part.equalsIgnoreCase("YEAR")){ + dateValue = DateFormatUtils.format(publishedDate, "yyyy"); + } + if(part.equalsIgnoreCase("MONTH")){ + dateValue = DateFormatUtils.format(publishedDate, "MM"); + } + if(part.equalsIgnoreCase("DAY")){ + dateValue = DateFormatUtils.format(publishedDate, "dd"); + } + pLink.add(dateValue); + } + } + } + + permalink = String.join("/", pLink); + permalink = sanitize(permalink).concat("/"); + + boolean noExtensionUri = config.getBoolean(Keys.URI_NO_EXTENSION); + String noExtensionUriPrefix = config.getString(Keys.URI_NO_EXTENSION_PREFIX); + if (noExtensionUri && (noExtensionUriPrefix != null && noExtensionUriPrefix.trim().isEmpty())) { + String uri = permalink; + uri = uri + "/index.html"; + fileContents.put(Attributes.URI, uri); + } else { + permalink = permalink.substring(0, permalink.length() -1 ); + permalink = permalink + config.getString(Keys.OUTPUT_EXTENSION); + } + + } + + + return permalink; + } + + /** + * Replace the spaces with hyphens + * @return + */ + private String sanitize(String input){ + return input.replace(" ", "-"); + } + public String getPathToRoot(File sourceFile) { File rootPath = new File(contentPath); From 7aad1adddb8ff3f5601502416309f324f9eec961 Mon Sep 17 00:00:00 2001 From: "manik.magar" Date: Sun, 26 Jun 2016 16:29:22 -0400 Subject: [PATCH 2/3] Updated test cases and permalink generation logic with rootpath handling --- src/main/java/org/jbake/app/Crawler.java | 75 ++++++--- src/main/resources/default.properties | 4 +- src/test/java/org/jbake/app/CrawlerTest.java | 151 ++++++++++++++++-- .../content/blog/2013/second-post.html | 2 +- 4 files changed, 197 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/jbake/app/Crawler.java b/src/main/java/org/jbake/app/Crawler.java index 7977a07fd..d3c2e0704 100644 --- a/src/main/java/org/jbake/app/Crawler.java +++ b/src/main/java/org/jbake/app/Crawler.java @@ -59,6 +59,7 @@ public static interface Status { static final String ROOTPATH = "rootpath"; static final String ID = "id"; static final String NO_EXTENSION_URI = "noExtensionUri"; + static final String PERMALINK = "permalink"; } private static final Logger LOGGER = LoggerFactory.getLogger(Crawler.class); @@ -181,12 +182,12 @@ private void crawlSourceFile(final File sourceFile, final String sha1, final Str } } - buildPermalink(fileContents); - - - if (config.getBoolean(Keys.URI_NO_EXTENSION)) { + if (config.getBoolean(Keys.URI_NO_EXTENSION)) { fileContents.put(Attributes.NO_EXTENSION_URI, uri.replace("/index.html", "/")); } + + String permalink = buildPermalink(fileContents); + fileContents.put(Attributes.PERMALINK, permalink); ODocument doc = new ODocument(documentType); doc.fields(fileContents); @@ -201,26 +202,52 @@ private void crawlSourceFile(final File sourceFile, final String sha1, final Str /** * This function generates permalinks if they are enabled in configuration. * - * Sample configuration entry: permalink.pattern = + * Default pattern is /:filepath + * + * Conditions - + * 1. String ending with ':' is treated as static strings. For example, permalink = /:blogdata:/:filepath, will generate all urls as /blogdata/{actual file path} + * 2. :filepath is reserved to add actual source file path (relative to content root) + * 3. :filename is reserved to add name of source file. + * 4. If the keyword values is array then all values of array are used for generation. For example, if /:tags is used and post has two tags tagA, tagB then url would be /tagA/tagB + * 5. :YEAR, :MONTH, :DAY are reserved to pull related part of content published date. + * + * If uri.noExtension is enabled then permalink generation will use it to generate extension less urls. + * + * on front end, permalinks can be accessed as {content.permalink} * * @param fileContents * @author Manik Magar * @return */ private String buildPermalink(Map fileContents){ - String permalink = (String) fileContents.get(Attributes.URI); - - boolean permalinkEnabled = config.getBoolean("permalink"); - if (permalinkEnabled) { - - String pattern = config.getString("permalink.pattern"); + String permalink = ""; + String separator = File.separator; + String permalinkPattern = config.getString(Attributes.PERMALINK,"/:filepath"); + if(config.containsKey(Attributes.PERMALINK +"."+ fileContents.get(Attributes.TYPE))){ + permalinkPattern = config.getString(Attributes.PERMALINK +"."+ fileContents.get(Attributes.TYPE)); + } + if (Objects.nonNull(permalinkPattern) && !permalinkPattern.trim().isEmpty()) { + String pattern = permalinkPattern; + if(pattern.startsWith(":")) pattern = separator+pattern; String[] parts = pattern.split("/:"); List pLink = new ArrayList(); for (String part : parts){ part = part.trim().replace("/", ""); if (part.endsWith(":")){ pLink.add(part.replace(":", "")); + } else if(part.equalsIgnoreCase("filepath")) { + String path = FileUtil.asPath(fileContents.get(Attributes.FILE).toString()).replace(FileUtil.asPath( contentPath), ""); + path = FilenameUtils.removeExtension(path); + // strip off leading / to enable generating non-root based sites + if (path.startsWith("/")) { + path = path.substring(1, path.length()); + } + pLink.add(path); + } else if(part.equalsIgnoreCase("filename")) { + String sourcePath = (String) fileContents.get(Attributes.SOURCE_URI); + String fileName = FilenameUtils.getBaseName(sourcePath); + pLink.add(fileName); } else if(fileContents.containsKey(part)){ Object value = fileContents.get(part); if (value instanceof String){ @@ -228,7 +255,7 @@ private String buildPermalink(Map fileContents){ } else if (value.getClass().equals(String[].class)){ pLink.addAll(Arrays.asList((String[])value)); } - } else if (Arrays.asList("YEAR","MONTH","DAY").contains(part)) { + } else if (Arrays.asList("YEAR","MONTH","DAY").contains(part.toUpperCase())) { Date publishedDate = (Date) fileContents.get("date"); if(Objects.nonNull(publishedDate)){ String dateValue = null; @@ -246,20 +273,26 @@ private String buildPermalink(Map fileContents){ } } - permalink = String.join("/", pLink); - permalink = sanitize(permalink).concat("/"); - + permalink = String.join(separator, pLink); + permalink = sanitize(permalink).concat(separator); + String uri = permalink; boolean noExtensionUri = config.getBoolean(Keys.URI_NO_EXTENSION); - String noExtensionUriPrefix = config.getString(Keys.URI_NO_EXTENSION_PREFIX); - if (noExtensionUri && (noExtensionUriPrefix != null && noExtensionUriPrefix.trim().isEmpty())) { - String uri = permalink; - uri = uri + "/index.html"; - fileContents.put(Attributes.URI, uri); + if (noExtensionUri) { + uri = uri + "index.html"; } else { permalink = permalink.substring(0, permalink.length() -1 ); permalink = permalink + config.getString(Keys.OUTPUT_EXTENSION); + uri = permalink; + } + if(uri.startsWith("/")){ + uri = uri.substring(1); } - + fileContents.put(Attributes.URI, uri); + + //Calculate the root path based on the permalink + File permaFile = new File(contentPath,uri); + String rootPath = getPathToRoot(permaFile); + fileContents.put(Attributes.ROOTPATH,rootPath); } diff --git a/src/main/resources/default.properties b/src/main/resources/default.properties index 6e10c1b6e..0aa68ae86 100644 --- a/src/main/resources/default.properties +++ b/src/main/resources/default.properties @@ -88,4 +88,6 @@ db.path=cache # enable extension-less URI option? uri.noExtension=false # Set to a prefix path (starting with a slash) for which to generate extension-less URI's (i.e. a folder with index.html in) -uri.noExtension.prefix= \ No newline at end of file +uri.noExtension.prefix= +# set the default permalink to source filepath. +permalink=/:filepath \ No newline at end of file diff --git a/src/test/java/org/jbake/app/CrawlerTest.java b/src/test/java/org/jbake/app/CrawlerTest.java index c5d46f409..387d170d2 100644 --- a/src/test/java/org/jbake/app/CrawlerTest.java +++ b/src/test/java/org/jbake/app/CrawlerTest.java @@ -1,19 +1,13 @@ package org.jbake.app; +import static org.assertj.core.api.Assertions.assertThat; + import java.io.File; import java.io.IOException; import java.net.URISyntaxException; import java.net.URL; -import java.util.List; -import java.util.Map; - -import com.orientechnologies.orient.core.record.impl.ODocument; -import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery; - -import org.apache.commons.configuration.CompositeConfiguration; -import org.apache.commons.configuration.ConfigurationException; -import org.jbake.app.ConfigUtil.Keys; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.commons.configuration.CompositeConfiguration; @@ -22,18 +16,19 @@ import org.apache.commons.io.FilenameUtils; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; +import org.jbake.app.ConfigUtil.Keys; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import static org.assertj.core.api.Assertions.*; +import com.orientechnologies.orient.core.record.impl.ODocument; public class CrawlerTest { private CompositeConfiguration config; private ContentStore db; private File sourceFolder; - + @Before public void setup() throws Exception, IOException, URISyntaxException { URL sourceUrl = this.getClass().getResource("/"); @@ -42,7 +37,7 @@ public void setup() throws Exception, IOException, URISyntaxException { if (!sourceFolder.exists()) { throw new Exception("Cannot find sample data structure!"); } - + config = ConfigUtil.load(new File(this.getClass().getResource("/").getFile())); Assert.assertEquals(".html", config.getString(Keys.OUTPUT_EXTENSION)); db = DBUtil.createDataStore("memory", "documents"+System.currentTimeMillis()); @@ -124,4 +119,136 @@ public static RegexMatcher matches(String regex){ return new RegexMatcher(regex); } } + + + private Map getPermalinkPost(String permalinkPattern){ + config.setProperty("permalink", permalinkPattern); + Crawler crawler = new Crawler(db, sourceFolder, config); + crawler.crawl(new File(sourceFolder.getPath() + File.separator + config.getString(Keys.CONTENT_FOLDER))); + + List results = db.getPublishedPostsByTag("PermalinkTest"); + + assertThat(results.size()).isEqualTo(1); + + DocumentList list = DocumentList.wrap(results.iterator()); + Map content = list.getFirst(); + return content; + } + + @Test + public void testPermalinkFilePath(){ + + Map content = getPermalinkPost("/:filepath"); + + assertThat(content.get("uri")).isEqualTo("blog/2013/second-post.html"); + assertThat(content.get("uri")).isEqualTo(content.get("permalink")); + assertThat(content) + .containsKey(Crawler.Attributes.ROOTPATH) + .containsValue("../../"); + + } + + @Test + public void testPermalinkFilePathWithStatic(){ + + Map content = getPermalinkPost("/:data:/:filepath"); + + assertThat(content.get("uri")).isEqualTo("data/blog/2013/second-post.html"); + assertThat(content.get("uri")).isEqualTo(content.get("permalink")); + assertThat(content) + .containsKey(Crawler.Attributes.ROOTPATH) + .containsValue("../../../"); + } + + @Test + public void testPermalinkFilename(){ + + Map content = getPermalinkPost("/:blog:/:filename"); + + assertThat(content.get("uri")).isEqualTo("blog/second-post.html"); + assertThat(content.get("uri")).isEqualTo(content.get("permalink")); + assertThat(content) + .containsKey(Crawler.Attributes.ROOTPATH) + .containsValue("../"); + } + + @Test + public void testPermalinkWithDate(){ + + Map content = getPermalinkPost("/:blog:/:YEAR/:MONTH/:DAY/:filename"); + + assertThat(content.get("uri")).isEqualTo("blog/2013/02/28/second-post.html"); + assertThat(content.get("uri")).isEqualTo(content.get("permalink")); + assertThat(content) + .containsKey(Crawler.Attributes.ROOTPATH) + .containsValue("../../../../"); + } + + @Test + public void testPermalinkWithTagAndTitle(){ + + Map content = getPermalinkPost("/:data:/:tags/:title"); + + assertThat(content.get("uri")).isEqualTo("data/blog/PermalinkTest/Second-Post.html"); + assertThat(content.get("uri")).isEqualTo(content.get("permalink")); + assertThat(content) + .containsKey(Crawler.Attributes.ROOTPATH) + .containsValue("../../../"); + } + + + @Test + public void testPermalinkWithTagAndTitleWithNoExtension(){ + config.setProperty("uri.noExtension", true); + Map content = getPermalinkPost("/:data:/:tags/:title"); + + assertThat(content.get("uri")).isEqualTo("data/blog/PermalinkTest/Second-Post/index.html"); + assertThat(content.get("permalink")).isEqualTo("data/blog/PermalinkTest/Second-Post/"); + assertThat(content) + .containsKey(Crawler.Attributes.ROOTPATH) + .containsValue("../../../../"); + } + + @Test + public void testPermalinkWithTypePermalink(){ + config.setProperty("permalink", "/:filepath"); + config.setProperty("permalink.post", "/:blog:/:filename"); + Map content = getPermalinkPost("/:filepath"); + + assertThat(content.get("uri")).isEqualTo("blog/second-post.html"); + assertThat(content.get("uri")).isEqualTo(content.get("permalink")); + assertThat(content) + .containsKey(Crawler.Attributes.ROOTPATH) + .containsValue("../"); + } + + @Test + public void testPermalinkWithMultipleTypePermalink(){ + config.setProperty("permalink", "/:filepath"); + config.setProperty("permalink.post", "/:blog:/:filename"); + config.setProperty("permalink.page", "/:pages:/:filename"); + Crawler crawler = new Crawler(db, sourceFolder, config); + crawler.crawl(new File(sourceFolder.getPath() + File.separator + config.getString(Keys.CONTENT_FOLDER))); + + List results = db.getPublishedPostsByTag("PermalinkTest"); + + assertThat(results.size()).isEqualTo(1); + + DocumentList list = DocumentList.wrap(results.iterator()); + Map content = list.getFirst(); + + //Verify that post has used permalink.post pattern. + assertThat(content.get("uri")).isEqualTo("blog/second-post.html"); + assertThat(content.get("uri")).isEqualTo(content.get("permalink")); + + List pageResults = db.getPublishedPages(); + + DocumentList pages = DocumentList.wrap(pageResults.iterator()); + //Verify that page has used permalink.page pattern. + Map page = pages.getFirst(); + String url = "pages/"+FilenameUtils.getBaseName((String) page.get("file")) + ".html"; + assertThat(page.get("uri")).isEqualTo(url); + assertThat(page.get("uri")).isEqualTo(page.get("permalink")); + + } } diff --git a/src/test/resources/content/blog/2013/second-post.html b/src/test/resources/content/blog/2013/second-post.html index 05d6d7d42..f67d0dda9 100644 --- a/src/test/resources/content/blog/2013/second-post.html +++ b/src/test/resources/content/blog/2013/second-post.html @@ -1,7 +1,7 @@ title=Second Post date=2013-02-28 type=post -tags=blog +tags=blog,PermalinkTest status=published og={"description": "Something"} ~~~~~~ From cd22f1871b09bb4b40d53c8c57a5ab0dacfeb55d Mon Sep 17 00:00:00 2001 From: "manik.magar" Date: Fri, 8 Jul 2016 19:41:29 -0400 Subject: [PATCH 3/3] Allow to override permalink in content --- src/main/java/org/jbake/app/Crawler.java | 88 +++++++++++++----------- 1 file changed, 48 insertions(+), 40 deletions(-) diff --git a/src/main/java/org/jbake/app/Crawler.java b/src/main/java/org/jbake/app/Crawler.java index 0926723b2..b12c162d4 100644 --- a/src/main/java/org/jbake/app/Crawler.java +++ b/src/main/java/org/jbake/app/Crawler.java @@ -222,6 +222,7 @@ private void crawlSourceFile(final File sourceFile, final String sha1, final Str */ private String buildPermalink(Map fileContents){ String permalink = ""; + String separator = File.separator; String permalinkPattern = config.getString(Attributes.PERMALINK,"/:filepath"); if(config.containsKey(Attributes.PERMALINK +"."+ fileContents.get(Attributes.TYPE))){ @@ -233,46 +234,53 @@ private String buildPermalink(Map fileContents){ if(pattern.startsWith(":")) pattern = separator+pattern; String[] parts = pattern.split("/:"); List pLink = new ArrayList(); - for (String part : parts){ - part = part.trim().replace("/", ""); - if (part.endsWith(":")){ - pLink.add(part.replace(":", "")); - } else if(part.equalsIgnoreCase("filepath")) { - String path = FileUtil.asPath(fileContents.get(Attributes.FILE).toString()).replace(FileUtil.asPath( contentPath), ""); - path = FilenameUtils.removeExtension(path); - // strip off leading / to enable generating non-root based sites - if (path.startsWith("/")) { - path = path.substring(1, path.length()); - } - pLink.add(path); - } else if(part.equalsIgnoreCase("filename")) { - String sourcePath = (String) fileContents.get(Attributes.SOURCE_URI); - String fileName = FilenameUtils.getBaseName(sourcePath); - pLink.add(fileName); - } else if(fileContents.containsKey(part)){ - Object value = fileContents.get(part); - if (value instanceof String){ - pLink.add(value.toString()); - } else if (value.getClass().equals(String[].class)){ - pLink.addAll(Arrays.asList((String[])value)); - } - } else if (Arrays.asList("YEAR","MONTH","DAY").contains(part.toUpperCase())) { - Date publishedDate = (Date) fileContents.get("date"); - if(publishedDate != null){ - String dateValue = null; - if(part.equalsIgnoreCase("YEAR")){ - dateValue = DateFormatUtils.format(publishedDate, "yyyy"); - } - if(part.equalsIgnoreCase("MONTH")){ - dateValue = DateFormatUtils.format(publishedDate, "MM"); - } - if(part.equalsIgnoreCase("DAY")){ - dateValue = DateFormatUtils.format(publishedDate, "dd"); - } - pLink.add(dateValue); - } - } - } + + + //Check if permalink is specified in the content, Use it as final link + if(fileContents.containsKey(Attributes.PERMALINK) && !StringUtils.isBlank(fileContents.get(Attributes.PERMALINK).toString())){ + pLink.add(fileContents.get(Attributes.PERMALINK).toString()); + } else { + for (String part : parts){ + part = part.trim().replace("/", ""); + if (part.endsWith(":")){ + pLink.add(part.replace(":", "")); + } else if(part.equalsIgnoreCase("filepath")) { + String path = FileUtil.asPath(fileContents.get(Attributes.FILE).toString()).replace(FileUtil.asPath( contentPath), ""); + path = FilenameUtils.removeExtension(path); + // strip off leading / to enable generating non-root based sites + if (path.startsWith("/")) { + path = path.substring(1, path.length()); + } + pLink.add(path); + } else if(part.equalsIgnoreCase("filename")) { + String sourcePath = (String) fileContents.get(Attributes.SOURCE_URI); + String fileName = FilenameUtils.getBaseName(sourcePath); + pLink.add(fileName); + } else if(fileContents.containsKey(part)){ + Object value = fileContents.get(part); + if (value instanceof String){ + pLink.add(value.toString()); + } else if (value.getClass().equals(String[].class)){ + pLink.addAll(Arrays.asList((String[])value)); + } + } else if (Arrays.asList("YEAR","MONTH","DAY").contains(part.toUpperCase())) { + Date publishedDate = (Date) fileContents.get("date"); + if(publishedDate != null){ + String dateValue = null; + if(part.equalsIgnoreCase("YEAR")){ + dateValue = DateFormatUtils.format(publishedDate, "yyyy"); + } + if(part.equalsIgnoreCase("MONTH")){ + dateValue = DateFormatUtils.format(publishedDate, "MM"); + } + if(part.equalsIgnoreCase("DAY")){ + dateValue = DateFormatUtils.format(publishedDate, "dd"); + } + pLink.add(dateValue); + } + } + } + } permalink = StringUtils.join(pLink, separator); permalink = sanitize(permalink).concat(separator);