feat: export images (#3)

basic logic for exporting all images that are referenced in markdown images (![alt](../assets/image.png))

README.md

@@ -20,10 +20,29 @@ logseq-export
         [MANDATORY] Folder where all public pages are exported.
   -graphPath string
         [MANDATORY] Path to the root of your logseq graph containing /pages and /journals directories.
+  -assetsRelativePath
+        relative path within blogFolder where the assets (images) should be stored (e.g. 'static/images/logseq'). Default is logseq-images (default "logseq-images")
+  -webAssetsPathPrefix
+    	  path that the images are going to be served on on the web (e.g. '/public/images/logseq'). Default is /logseq-images (default "/logseq-images")
   -unquotedProperties string
         comma-separated list of logseq page properties that won't be quoted in the markdown frontmatter, e.g. 'date,public,slug'
 ```
 
+#### Command example
+
+This is how I run the command on my machine:
+
+```sh
+logseq-export \
+  --graphPath /Users/tomas/workspace/private/notes \
+  --blogFolder /Users/tomas/workspace/private/blog \
+  --unquotedProperties date,slug,public,tags \
+  --assetsRelativePath static/images/logseq \
+  --webAssetsPathPrefix /images/logseq
+```
+
+This will take my logseq notes and copies them to blog, it will also copy all the images to `/Users/tomas/workspace/private/blog/static/images/logseq`, but the image links themselves are going to have `/images/logseq` prefix (`![alt](/images/logseq/image.png)`).
+
 ### Logseq page properties with a special meaning (all optional)
 
 - `public` - as soon as this page property is present (regardless of value), the page gets exported

file_utils.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"io"
+	"os"
+)
+
+func readFileToString(src string) (string, error) {
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return "", err
+	}
+	defer srcFile.Close()
+	bytes, err := os.ReadFile(src)
+	if err != nil {
+		return "", err
+	}
+	return string(bytes), nil
+}
+
+func writeStringToFile(dest string, content string) error {
+	err := os.WriteFile(dest, []byte(content), os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func copy(src, dest string) error {
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer srcFile.Close()
+
+	destFile, err := os.Create(dest) // creates if file doesn't exist
+	if err != nil {
+		return err
+	}
+	defer destFile.Close()
+
+	_, err = io.Copy(destFile, srcFile) // check first var for number of bytes copied
+	if err != nil {
+		return err
+	}
+
+	err = destFile.Sync()
+	if err != nil {
+		return err
+	}
+	return nil
+}

main.go

@@ -16,6 +16,7 @@ import (
 type page struct {
 	filename   string
 	attributes map[string]string
+	assets     []string
 	text       string
 }
 
@@ -52,6 +53,8 @@ func findMatchingFiles(rootPath string, substring string) ([]string, error) {
 func main() {
 	graphPath := flag.String("graphPath", "", "[MANDATORY] Folder where all public pages are exported.")
 	blogFolder := flag.String("blogFolder", "", "[MANDATORY] Folder where this program creates a new subfolder with public logseq pages.")
+	assetsRelativePath := flag.String("assetsRelativePath", "logseq-images", "relative path within blogFolder where the assets (images) should be stored (e.g. 'static/images/logseq'). Default is `logseq-images`")
+	webAssetsPathPrefix := flag.String("webAssetsPathPrefix", "/logseq-images", "path that the images are going to be served on on the web (e.g. '/public/images/logseq'). Default is `/logseq-images`")
 	unquotedProperties := flag.String("unquotedProperties", "", "comma-separated list of logseq page properties that won't be quoted in the markdown front matter, e.g. 'date,public,slug")
 	flag.Parse()
 	if *graphPath == "" || *blogFolder == "" {
@@ -70,7 +73,12 @@ func main() {
 		}
 		_, name := filepath.Split(publicFile)
 		page := parsePage(name, srcContent)
-		result := transformPage(page)
+		result := transformPage(page, *webAssetsPathPrefix)
+		assetFolder := filepath.Join(*blogFolder, *assetsRelativePath)
+		err = copyAssets(publicFile, assetFolder, result.assets)
+		if err != nil {
+			log.Fatalf("Error when copying assets for page %q: %v", publicFile, err)
+		}
 		dest := filepath.Join(*blogFolder, result.filename)
 		folder, _ := filepath.Split(dest)
 		err = os.MkdirAll(folder, os.ModePerm)
@@ -84,6 +92,24 @@ func main() {
 	}
 }
 
+func copyAssets(baseFile string, assetFolder string, assets []string) error {
+	err := os.MkdirAll(assetFolder, os.ModePerm)
+	if err != nil {
+		log.Fatalf("Error when making assets folder %q: %v", assetFolder, err)
+	}
+	baseDir, _ := filepath.Split(baseFile)
+	for _, relativeAssetPath := range assets {
+		assetPath := filepath.Clean(filepath.Join(baseDir, relativeAssetPath))
+		_, assetName := filepath.Split(assetPath)
+		destPath := filepath.Join(assetFolder, assetName)
+		err := copy(assetPath, destPath)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func parseUnquotedProperties(param string) []string {
 	if param == "" {
 		return []string{}
@@ -107,25 +133,3 @@ func render(p page, dontQuote []string) string {
 	}
 	return fmt.Sprintf("---\n%s---\n%s", attributeBuilder.String(), p.text)
 }
-
-func readFileToString(src string) (string, error) {
-	srcFile, err := os.Open(src)
-	if err != nil {
-		return "", err
-	}
-	defer srcFile.Close()
-	bytes, err := os.ReadFile(src)
-	if err != nil {
-		return "", err
-	}
-	return string(bytes), nil
-}
-
-func writeStringToFile(dest string, content string) error {
-	err := os.WriteFile(dest, []byte(content), os.ModePerm)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

transform.go

@@ -25,6 +25,13 @@ func generateFileName(originalName string, attributes map[string]string) string
 	return fmt.Sprintf("%s%s.md", date, attributes["slug"])
 }
 
+func addFileName(p page) page {
+	filename := generateFileName(p.filename, p.attributes)
+	folder := filepath.Join(path.Split(p.attributes["folder"])) // the page property always uses `/` but the final delimiter is OS-dependent
+	p.filename = filepath.Join(folder, filename)
+	return p
+}
+
 func removeEmptyBulletPoints(from string) string {
 	return regexp.MustCompile(`(?m:^\s*-\s*$)`).ReplaceAllString(from, "")
 }
@@ -68,28 +75,57 @@ func unindentMultilineStrings(from string) string {
 	})
 }
 
-func applyAll(from string, transformers ...func(string) string) string {
+// onlyText turns text transformer into a page transformer
+func onlyText(textTransformer func(string) string) func(page) page {
+	return func(p page) page {
+		p.text = textTransformer(p.text)
+		return p
+	}
+}
+
+func applyAll(from page, transformers ...func(page) page) page {
 	result := from
 	for _, t := range transformers {
 		result = t(result)
 	}
 	return result
 }
 
-func transformPage(p page) page {
-	filename := generateFileName(p.filename, p.attributes)
-	folder := filepath.Join(path.Split(p.attributes["folder"])) // the page property always uses `/` but the final delimiter is OS-dependent
-	text := applyAll(
-		p.text,
-		removeEmptyBulletPoints,
-		unindentMultilineStrings,
-		firstBulletPointsToParagraphs,
-		secondToFirstBulletPoints,
-		removeTabFromMultiLevelBulletPoints,
-	)
-	return page{
-		filename:   filepath.Join(folder, filename),
-		attributes: p.attributes,
-		text:       text,
+/*
+extractAssets finds all markdown images with **relative** URL e.g. `![alt](../assets/image.png)`
+it extracts the relative URL into a `page.assets“ array
+it replaces the relative links with `imagePrefixPath“: `{imagePrefixPath}/image.png`
+*/
+func extractAssets(imagePrefixPath string) func(page) page {
+	return func(p page) page {
+		assetRegexp := regexp.MustCompile(`!\[.*?]\((\.\.?/.+?)\)`)
+		links := assetRegexp.FindAllStringSubmatch(p.text, -1)
+		assets := make([]string, 0, len(links))
+		for _, l := range links {
+			assets = append(assets, l[1])
+		}
+		p.assets = assets
+		textWithAssets := assetRegexp.ReplaceAllStringFunc(p.text, func(s string) string {
+			match := assetRegexp.FindStringSubmatch(s)
+			originalURL := match[1]
+			_, assetName := path.Split(originalURL)
+			newURL := path.Join(imagePrefixPath, assetName)
+			return strings.Replace(s, originalURL, newURL, 1)
+		})
+		p.text = textWithAssets
+		return p
 	}
 }
+
+func transformPage(p page, webAssetsPathPrefix string) page {
+	return applyAll(
+		p,
+		addFileName,
+		onlyText(removeEmptyBulletPoints),
+		onlyText(unindentMultilineStrings),
+		onlyText(firstBulletPointsToParagraphs),
+		onlyText(secondToFirstBulletPoints),
+		onlyText(removeTabFromMultiLevelBulletPoints),
+		extractAssets(webAssetsPathPrefix),
+	)
+}

transform_test.go

@@ -43,7 +43,7 @@ func transformText(from string) string {
 		attributes: map[string]string{},
 		text:       from,
 	}
-	result := transformPage(testPage)
+	result := transformPage(testPage, "")
 	return result.text
 }
 
@@ -57,7 +57,7 @@ func TestTransformPage(t *testing.T) {
 			},
 			text: "",
 		}
-		result := transformPage(testPage)
+		result := transformPage(testPage, "")
 		require.Equal(t, "2022-09-24-this-is-a-slug.md", result.filename)
 	})
 
@@ -69,7 +69,7 @@ func TestTransformPage(t *testing.T) {
 			},
 			text: "",
 		}
-		result := transformPage(testPage)
+		result := transformPage(testPage, "")
 		require.Equal(t, filepath.Join("content", "posts", "name-with-space.md"), result.filename)
 	})
 
@@ -116,3 +116,24 @@ in
 one`, result)
 	})
 }
+
+func TestTransformImages(t *testing.T) {
+	t.Run("extracts relative images", func(t *testing.T) {
+		testPage := page{
+			filename: "a.md",
+			text:     "- ![hello world](../assets/image.png)",
+		}
+		result := transformPage(testPage, "/images")
+		require.Equal(t, []string{"../assets/image.png"}, result.assets)
+		require.Equal(t, "\n![hello world](/images/image.png)", result.text)
+	})
+	t.Run("ignores absolute images", func(t *testing.T) {
+		testPage := page{
+			filename: "a.md",
+			text:     "- ![hello world](http://example.com/assets/image.png)",
+		}
+		result := transformPage(testPage, "/images")
+		require.Equal(t, 0, len(result.assets))
+		require.Equal(t, "\n![hello world](http://example.com/assets/image.png)", result.text)
+	})
+}