diff --git a/hugolib/page_test.go b/hugolib/page_test.go index 66afd7d966e..429ab26599a 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -593,7 +593,7 @@ func TestPageSummary(t *testing.T) { // Source is not Asciidoctor- or RST-compatible so don't test them if ext != "ad" && ext != "rst" { checkPageContent(t, p, normalizeExpected(ext, "

Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

\n\n

Additional text.

\n\n

Further text.

\n"), ext) - checkPageSummary(t, p, normalizeExpected(ext, "

Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

"), ext) + checkPageSummary(t, p, normalizeExpected(ext, "

Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

Additional text.

"), ext) } checkPageType(t, p, "page") } diff --git a/resources/page/page_markup.go b/resources/page/page_markup.go index ef4a56e3a74..f600a06ef0b 100644 --- a/resources/page/page_markup.go +++ b/resources/page/page_markup.go @@ -161,6 +161,16 @@ func (s *HtmlSummary) resolveParagraphTagAndSetWrapper(mt media.Type) tagReStart return ptag } +// Avoid counting words that are most likely HTML tokens. +var ( + isProbablyHTMLTag = regexp.MustCompile(`^<\/?[A-Za-z]+>?$`) + isProablyHTMLAttribute = regexp.MustCompile(`^[A-Za-z]+=["']`) +) + +func isProbablyHTMLToken(s string) bool { + return isProbablyHTMLTag.MatchString(s) || isProablyHTMLAttribute.MatchString(s) || s == ">" +} + // ExtractSummaryFromHTML extracts a summary from the given HTML content. func ExtractSummaryFromHTML(mt media.Type, input string, numWords int, isCJK bool) (result HtmlSummary) { result.source = input @@ -173,6 +183,14 @@ func ExtractSummaryFromHTML(mt media.Type, input string, numWords int, isCJK boo var count int countWord := func(word string) int { + word = strings.TrimSpace(word) + if len(word) == 0 { + return 0 + } + if isProbablyHTMLToken(word) { + return 0 + } + if isCJK { word = tpl.StripHTML(word) runeCount := utf8.RuneCountInString(word) @@ -193,7 +211,7 @@ func ExtractSummaryFromHTML(mt media.Type, input string, numWords int, isCJK boo for j := result.WrapperStart.High; j < high; { s := input[j:] - closingIndex := strings.Index(s, "") if closingIndex == -1 { break diff --git a/resources/page/page_markup_test.go b/resources/page/page_markup_test.go index b7d363f8f68..43eaae6f6ad 100644 --- a/resources/page/page_markup_test.go +++ b/resources/page/page_markup_test.go @@ -49,6 +49,46 @@ func TestExtractSummaryFromHTML(t *testing.T) { } } +// See https://discourse.gohugo.io/t/automatic-summarys-summarylength-seems-broken-in-the-case-of-plainify/51466/4 +// Also issue 12837 +func TestExtractSummaryFromHTMLLotsOfHTMLInSummary(t *testing.T) { + c := qt.New(t) + + input := ` +

+

+ + 1 + + + 2 + + + 3 + + + 4 + + + 5 + +
+

+

+This is a story about a cat. +

+

+The cat was white and fluffy. +

+

+And it liked milk. +

+` + + summary := ExtractSummaryFromHTML(media.Builtin.MarkdownType, input, 10, false) + c.Assert(strings.HasSuffix(summary.Summary(), "

\nThis is a story about a cat.\n

\n

\nThe cat was white and fluffy.\n

"), qt.IsTrue) +} + func TestExtractSummaryFromHTMLWithDivider(t *testing.T) { c := qt.New(t) @@ -114,6 +154,23 @@ func TestExpandDivider(t *testing.T) { } } +func TestIsProbablyHTMLToken(t *testing.T) { + c := qt.New(t) + + for i, test := range []struct { + input string + expect bool + }{ + {"

", true}, + {"Æøå", false}, + } { + c.Assert(isProbablyHTMLToken(test.input), qt.Equals, test.expect, qt.Commentf("[%d] Test.expect %q", i, test.input)) + } +} + func BenchmarkSummaryFromHTML(b *testing.B) { b.StopTimer() input := "

First paragraph

Second paragraph

"