Added support for formatting in-md HTML and comments.

Signed-off-by: Bartlomiej Plotka <[email protected]>

go.mod

@@ -1,6 +1,6 @@
 module github.com/bwplotka/mdox
 
-go 1.14
+go 1.15
 
 require (
 	github.com/Kunde21/markdownfmt/v2 v2.1.0
@@ -16,10 +16,14 @@ require (
 	github.com/oklog/run v1.1.0
 	github.com/pkg/errors v0.9.1
 	github.com/sergi/go-diff v1.0.0
-	github.com/yuin/goldmark v1.3.1
+	github.com/yuin/goldmark v1.3.5
 	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
+	golang.org/x/net v0.0.0-20200822124328-c89045814202
 	golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 // indirect
 	golang.org/x/tools v0.0.0-20201020161133-226fd2f889ca // indirect
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
 )
+
+// TODO(bwplotka): Remove when https://github.com/Kunde21/markdownfmt/pull/35 is merged.
+replace github.com/Kunde21/markdownfmt/v2 => github.com/bwplotka/markdownfmt/v2 v2.0.0-20210616121647-559e77044d46

go.sum

@@ -26,8 +26,6 @@ github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo
 github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
 github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20190418212003-6ac0b49e7197/go.mod h1:aJ4qN3TfrelA6NZ6AXsXRfmEVaYin3EDbSPJrKS8OXo=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
-github.com/Kunde21/markdownfmt/v2 v2.1.0 h1:lzLCOy/4xFJwWzyF2o1U+RGJb+GoOVWJ6hNfoew2HPc=
-github.com/Kunde21/markdownfmt/v2 v2.1.0/go.mod h1:XK8I51ZbmTEjRizk8Uzu2sC3h8F0x4PM3RXGkNXObkA=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
 github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
@@ -88,6 +86,8 @@ github.com/bep/gitmap v1.1.2/go.mod h1:g9VRETxFUXNWzMiuxOwcudo6DfZkW9jOsOW0Ft4kY
 github.com/bep/golibsass v0.6.0/go.mod h1:DL87K8Un/+pWUS75ggYv41bliGiolxzDKWJAq3eJ1MA=
 github.com/bep/tmc v0.5.1/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bwplotka/markdownfmt/v2 v2.0.0-20210616121647-559e77044d46 h1:rSyClaQH3A1TVNC3jUOwSkJw4V8rwUDsAZIJEDH6ONQ=
+github.com/bwplotka/markdownfmt/v2 v2.0.0-20210616121647-559e77044d46/go.mod h1:LFJueuHZej/Z7Xhqh/XgClfkDjZiiEBOLVTt1Duq1r0=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -470,8 +470,8 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:
 github.com/yuin/goldmark v1.1.22/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.31/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.3.1 h1:eVwehsLsZlCJCwXyGLgg+Q4iFWE/eTIMG0e8waCmm/I=
-github.com/yuin/goldmark v1.3.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark-highlighting v0.0.0-20200307114337-60d527fdb691/go.mod h1:YLF3kDffRfUH/bTxOxHhV6lxwIB3Vfj91rEwNMS9MXo=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
@@ -516,7 +516,6 @@ golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

pkg/mdformatter/mdformatter_test.go

@@ -91,6 +91,7 @@ func TestFormat_FormatSingle_Transformers(t *testing.T) {
 	t.Run("Format not formatted", func(t *testing.T) {
 		buf := bytes.Buffer{}
 		testutil.Ok(t, f.Format(file, &buf))
+
 		testutil.Equals(t, string(exp), buf.String())
 	})
 

pkg/mdformatter/testdata/formatted.md

@@ -227,6 +227,41 @@ The compactor is not in the critical path of querying or data backup. It can eit
 
 In case of Prometheus with Thanos sidecar does not have enough retention, or if you want to have alerts or recording rules that requires global view, Thanos has just the component for that: the [Ruler](components/rule.md), which does rule and alert evaluation on top of a given Thanos Querier.
 
+<!--- TODO explain steps  --->
+
+<img src="../img/go-in-thanos.jpg" class="img-fluid" alt="Go in Thanos" >
+
+<p align="center"><img src="docs/img/Thanos-logo_fullmedium.png" alt="Thanos Logo"></p>
+
+<table>
+<tbody>
+<tr><th>Avoid 🔥[Link](../docs/something.png)</th></tr>
+<tr><td>
+
+```go
+resp, err := http.Get("http://example.com/")
+if err != nil {
+    // handle...
+}
+defer runutil.CloseWithLogOnErr(logger, resp.Body, "close response")
+
+scanner := bufio.NewScanner(resp.Body)
+// If any error happens and we return in the middle of scanning
+// body, we can end up with unread buffer, which
+// will use memory and hold TCP connection!
+for scanner.Scan() {
+```
+
+</td></tr>
+<tr><th>Better 🤓</th></tr>
+</tbody>
+</table>
+
+<dsada
+
+<taasdav>
+</taasdav>
+
 ## Flags
 
 ```$

pkg/mdformatter/testdata/formatted_and_transformed.md

@@ -49,7 +49,7 @@ Following the [KISS]($$-https://en.wikipedia.org/wiki/KISS_principle-testdata/no
 
 See those components on this diagram:
 
-<img src="img/arch.jpg" class="img-fluid" alt="architecture overview" />
+<img src="$$-img/arch.jpg-testdata/not_formatted.md-$$" class="img-fluid" alt="architecture overview"/>
 
 ![img]($$-img/arch.jpg-testdata/not_formatted.md-$$)
 
@@ -227,6 +227,41 @@ The compactor is not in the critical path of querying or data backup. It can eit
 
 In case of Prometheus with Thanos sidecar does not have enough retention, or if you want to have alerts or recording rules that requires global view, Thanos has just the component for that: the [Ruler]($$-components/rule.md-testdata/not_formatted.md-$$), which does rule and alert evaluation on top of a given Thanos Querier.
 
+<!--- TODO explain steps  --->
+
+<img src="$$-../img/go-in-thanos.jpg-testdata/not_formatted.md-$$" class="img-fluid" alt="Go in Thanos">
+
+<p align="center"><img src="$$-docs/img/Thanos-logo_fullmedium.png-testdata/not_formatted.md-$$" alt="Thanos Logo"></p>
+
+<table>
+<tbody>
+<tr><th>Avoid 🔥[Link](../docs/something.png)</th></tr>
+<tr><td>
+
+```go
+resp, err := http.Get("http://example.com/")
+if err != nil {
+    // handle...
+}
+defer runutil.CloseWithLogOnErr(logger, resp.Body, "close response")
+
+scanner := bufio.NewScanner(resp.Body)
+// If any error happens and we return in the middle of scanning
+// body, we can end up with unread buffer, which
+// will use memory and hold TCP connection!
+for scanner.Scan() {
+```
+
+</td></tr>
+<tr><th>Better 🤓</th></tr>
+</tbody>
+</table>
+
+<dsada
+
+<taasdav>
+</taasdav>
+
 ## Flags
 
 ```$

pkg/mdformatter/testdata/not_formatted.md

@@ -237,6 +237,40 @@ In case of Prometheus with Thanos sidecar does not have enough retention, or if
 which does rule and alert evaluation on top of a given Thanos Querier.
 
 
+<!--- TODO explain steps  --->
+
+<img src="../img/go-in-thanos.jpg" class="img-fluid" alt="Go in Thanos" >
+
+<p align="center"><img src="docs/img/Thanos-logo_fullmedium.png" alt="Thanos Logo"></p>
+
+<table>
+<tbody>
+<tr><th>Avoid 🔥[Link](../docs/something.png)</th></tr>
+<tr><td>
+
+```go
+resp, err := http.Get("http://example.com/")
+if err != nil {
+    // handle...
+}
+defer runutil.CloseWithLogOnErr(logger, resp.Body, "close response")
+
+scanner := bufio.NewScanner(resp.Body)
+// If any error happens and we return in the middle of scanning
+// body, we can end up with unread buffer, which
+// will use memory and hold TCP connection!
+for scanner.Scan() {
+```
+
+</td></tr>
+<tr><th>Better 🤓</th></tr>
+</tbody>
+</table>
+
+<dsada
+
+<taasdav>
+</taasdav>
 
 ## Flags
 

pkg/mdformatter/testdata/not_formatted.md.diff

@@ -115,7 +115,7 @@
 
  ### [Compactor](components/compact.md)
 
-@@ -226,8 +218,4 @@
+@@ -226,5 +218,3 @@
 
  The compactor is not in the critical path of querying or data backup. It can either be run as a periodic batch job or be left running to always compact data as soon as possible. It is recommended to provide 100-300GB of local disk space for data processing.
 
@@ -132,23 +132,25 @@
 -
 +In case of Prometheus with Thanos sidecar does not have enough retention, or if you want to have alerts or recording rules that requires global view, Thanos has just the component for that: the [Ruler](components/rule.md), which does rule and alert evaluation on top of a given Thanos Querier.
 
--
+ <!--- TODO explain steps  --->
+
+@@ -273,1 +263,0 @@
+
  ## Flags
--
+
 -[embedmd]:# (flags/rule.txt $)
-+
  ```$
  usage: thanos rule [<flags>]
 
-@@ -414,1 +402,0 @@
+@@ -448,1 +437,0 @@
 
  The configuration format is the following:
 
 -[embedmd]:# (../flags/config_rule_alerting.txt yaml)
  ```yaml
  alertmanagers:
  - http_config:
-@@ -440,1 +427,4 @@
+@@ -474,1 +462,4 @@
    timeout: 10s
    api_version: v1
  ```

pkg/mdformatter/transformer.go

@@ -11,6 +11,7 @@ import (
 	"github.com/yuin/goldmark/ast"
 	"github.com/yuin/goldmark/renderer"
 	"github.com/yuin/goldmark/text"
+	"golang.org/x/net/html"
 )
 
 type nopOpsRenderer struct {
@@ -40,7 +41,72 @@ func (t *transformer) Render(w io.Writer, source []byte, node ast.Node) error {
 	if err := ast.Walk(node, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
 		var err error
 		switch typedNode := n.(type) {
-		// TODO(bwplotka): Add support for links inside HTML.
+		case *ast.HTMLBlock, *ast.RawHTML:
+			if !entering || t.link == nil {
+				return ast.WalkSkipChildren, nil
+			}
+
+			// Parse HTML to get inline links on our own, goldmark does not do that.
+			b := bytes.Buffer{}
+			if typedNode, ok := n.(*ast.RawHTML); ok {
+				for i := 0; i < typedNode.Segments.Len(); i++ {
+					segment := typedNode.Segments.At(i)
+					_, _ = b.Write(segment.Value(source))
+				}
+			} else {
+				// We switch this to string type so we need to accommodate newlines.
+				_, _ = b.WriteString("\n")
+				if n.HasBlankPreviousLines() {
+					_, _ = b.WriteString("\n")
+				}
+				for i := 0; i < n.Lines().Len(); i++ {
+					segment := n.Lines().At(i)
+					_, _ = b.Write(segment.Value(source))
+				}
+			}
+
+			var out string
+			z := html.NewTokenizer(&b)
+			for tt := z.Next(); tt != html.ErrorToken; tt = z.Next() {
+				token := z.Token()
+				switch token.Data {
+				case "img":
+					for i := range token.Attr {
+						if token.Attr[i].Key != "src" {
+							continue
+						}
+						dest, err := t.link.TransformDestination(t.sourceCtx, []byte(token.Attr[i].Val))
+						if err != nil {
+							return ast.WalkStop, err
+						}
+						token.Attr[i].Val = string(dest)
+						break
+					}
+				case "a":
+					for i := range token.Attr {
+						if token.Attr[i].Key != "href" {
+							continue
+						}
+						dest, err := t.link.TransformDestination(t.sourceCtx, []byte(token.Attr[i].Val))
+						if err != nil {
+							return ast.WalkStop, err
+						}
+						token.Attr[i].Val = string(dest)
+						break
+					}
+				}
+				out += token.String()
+			}
+			if err := z.Err(); err != nil && err != io.EOF {
+				return ast.WalkStop, err
+			}
+
+			repl := ast.NewString([]byte("\n" + out + "\n"))
+			repl.SetParent(n.Parent())
+			repl.SetPreviousSibling(n.PreviousSibling())
+			repl.SetNextSibling(n.NextSibling())
+			n.Parent().ReplaceChild(n.Parent(), n, repl)
+			n.SetNextSibling(repl.NextSibling()) // Make sure our loop can continue.
 		case *ast.Link:
 			if !entering || t.link == nil {
 				return ast.WalkSkipChildren, nil
@@ -63,6 +129,8 @@ func (t *transformer) Render(w io.Writer, source []byte, node ast.Node) error {
 			repl := ast.NewString(dest)
 			repl.SetParent(n)
 			n.Parent().ReplaceChild(n.Parent(), n, repl)
+			n.SetNextSibling(repl.NextSibling()) // Make sure our loop can continue.
+
 		case *ast.Image:
 			if !entering || t.link == nil {
 				return ast.WalkSkipChildren, nil