Merge pull request #180 from walles/johan/force-format

Add a -lang option for setting the highlighting language

m/highlight.go

@@ -2,40 +2,18 @@ package m
 
 import (
 	"bytes"
-	"os"
 	"strings"
 
 	"github.com/alecthomas/chroma/v2"
-	"github.com/alecthomas/chroma/v2/lexers"
-
-	log "github.com/sirupsen/logrus"
 )
 
-// Files larger than this won't be highlighted
-//
-//revive:disable-next-line:var-naming
-const MAX_HIGHLIGHT_SIZE int64 = 1024 * 1024
-
-// Read and highlight a file using Chroma: https://github.com/alecthomas/chroma
+// Read and highlight some text using Chroma:
+// https://github.com/alecthomas/chroma
 //
-// If force is true, file will always be highlighted. If force is false, files
-// larger than MAX_HIGHLIGHT_SIZE will not be highlighted.
+// If lexer is nil no highlighting will be performed.
 //
 // Returns nil with no error if highlighting would be a no-op.
-func highlight(filename string, force bool, style chroma.Style, formatter chroma.Formatter) (*string, error) {
-	// Highlight input file using Chroma:
-	// https://github.com/alecthomas/chroma
-	fileInfo, err := os.Stat(filename)
-	if err != nil {
-		return nil, err
-	}
-	if fileInfo.Size() > MAX_HIGHLIGHT_SIZE && !force {
-		log.Debugf("Not highlighting %s because it is %d bytes large, which is larger than moar's built-in highlighting limit of %d bytes",
-			filename, fileInfo.Size(), MAX_HIGHLIGHT_SIZE)
-		return nil, nil
-	}
-
-	lexer := lexers.Match(filename)
+func highlight(text string, style chroma.Style, formatter chroma.Formatter, lexer chroma.Lexer) (*string, error) {
 	if lexer == nil {
 		// No highlighter available for this file type
 		return nil, nil
@@ -55,12 +33,7 @@ func highlight(filename string, force bool, style chroma.Style, formatter chroma
 	// with and without.
 	lexer = chroma.Coalesce(lexer)
 
-	contents, err := os.ReadFile(filename)
-	if err != nil {
-		return nil, err
-	}
-
-	iterator, err := lexer.Tokenise(nil, string(contents))
+	iterator, err := lexer.Tokenise(nil, text)
 	if err != nil {
 		return nil, err
 	}

m/pager_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/alecthomas/chroma/v2/formatters"
+	"github.com/alecthomas/chroma/v2/lexers"
 	"github.com/alecthomas/chroma/v2/styles"
 	"github.com/google/go-cmp/cmp"
 	"github.com/walles/moar/twin"
@@ -165,7 +166,7 @@ func TestCodeHighlighting(t *testing.T) {
 		panic("Getting current filename failed")
 	}
 
-	reader, err := NewReaderFromFilename(filename, *styles.Get("native"), formatters.TTY16m)
+	reader, err := NewReaderFromFilename(filename, *styles.Get("native"), formatters.TTY16m, nil)
 	if err != nil {
 		panic(err)
 	}
@@ -607,23 +608,22 @@ func benchmarkSearch(b *testing.B, highlighted bool) {
 		panic("Getting current filename failed")
 	}
 
+	sourceBytes, err := os.ReadFile(sourceFilename)
+	if err != nil {
+		panic(err)
+	}
+	fileContents := string(sourceBytes)
+
 	// Read one copy of the example input
-	var fileContents string
 	if highlighted {
-		highlightedSourceCode, err := highlight(sourceFilename, true, *styles.Get("native"), formatters.TTY16m)
+		highlightedSourceCode, err := highlight(fileContents, *styles.Get("native"), formatters.TTY16m, lexers.Get("go"))
 		if err != nil {
 			panic(err)
 		}
 		if highlightedSourceCode == nil {
 			panic("Highlighting didn't want to, returned nil")
 		}
 		fileContents = *highlightedSourceCode
-	} else {
-		sourceBytes, err := os.ReadFile(sourceFilename)
-		if err != nil {
-			panic(err)
-		}
-		fileContents = string(sourceBytes)
 	}
 
 	// Duplicate data N times

m/reader.go

@@ -15,9 +15,15 @@ import (
 	"time"
 
 	"github.com/alecthomas/chroma/v2"
+	"github.com/alecthomas/chroma/v2/lexers"
 	log "github.com/sirupsen/logrus"
 )
 
+// Files larger than this won't be highlighted
+//
+//revive:disable-next-line:var-naming
+const MAX_HIGHLIGHT_SIZE int64 = 1024 * 1024
+
 // Reader reads a file into an array of strings.
 //
 // It does the reading in the background, and it returns parts of the read data
@@ -407,10 +413,12 @@ func countLines(filename string) (uint64, error) {
 
 // NewReaderFromFilename creates a new file reader.
 //
+// If lexer is nil it will be determined from the input file name.
+//
 // The Reader will try to uncompress various compressed file format, and also
 // apply highlighting to the file using Chroma:
 // https://github.com/alecthomas/chroma
-func NewReaderFromFilename(filename string, style chroma.Style, formatter chroma.Formatter) (*Reader, error) {
+func NewReaderFromFilename(filename string, style chroma.Style, formatter chroma.Formatter, lexer chroma.Lexer) (*Reader, error) {
 	fileError := tryOpen(filename)
 	if fileError != nil {
 		return nil, fileError
@@ -436,18 +444,49 @@ func NewReaderFromFilename(filename string, style chroma.Style, formatter chroma
 	returnMe.name = &filename
 	returnMe.Unlock()
 
+	returnMe.StartHighlightingFromFile(filename, style, formatter, lexer)
+
+	return returnMe, nil
+}
+
+func (reader *Reader) StartHighlightingFromFile(filename string, style chroma.Style, formatter chroma.Formatter, lexer chroma.Lexer) {
+	reportDone := func() {
+		reader.highlightingDone.Store(true)
+		select {
+		case reader.maybeDone <- true:
+		default:
+		}
+
+		log.Trace("Highlighting done")
+	}
+
+	fileInfo, err := os.Stat(filename)
+	if err != nil {
+		log.Warn("Failed to stat file for highlighting: ", err)
+		reportDone()
+		return
+	}
+	if fileInfo.Size() > MAX_HIGHLIGHT_SIZE {
+		log.Debug("File too large for highlighting: ", fileInfo.Size())
+		reportDone()
+		return
+	}
+
 	go func() {
-		defer func() {
-			returnMe.highlightingDone.Store(true)
-			select {
-			case returnMe.maybeDone <- true:
-			default:
-			}
+		defer reportDone()
 
-			log.Trace("Highlighting done")
-		}()
+		fileBytes, err := os.ReadFile(filename)
+		if err != nil {
+			log.Warn("Failed to read file for highlighting: ", err)
+			return
+		}
 
-		highlighted, err := highlight(filename, false, style, formatter)
+		if lexer == nil {
+			// Try auto detecting by filename
+			lexer = lexers.Match(filename)
+		}
+
+		highlighted, err := highlight(string(fileBytes), style, formatter, lexer)
 		if err != nil {
 			log.Warn("Highlighting failed: ", err)
 			return
@@ -458,10 +497,8 @@ func NewReaderFromFilename(filename string, style chroma.Style, formatter chroma
 			return
 		}
 
-		returnMe.setText(*highlighted)
+		reader.setText(*highlighted)
 	}()
-
-	return returnMe, nil
 }
 
 // createStatusUnlocked() assumes that its caller is holding the lock

m/reader_test.go

@@ -171,7 +171,7 @@ func TestGetLines(t *testing.T) {
 			}
 		}
 
-		reader, err := NewReaderFromFilename(file, *styles.Get("native"), formatters.TTY16m)
+		reader, err := NewReaderFromFilename(file, *styles.Get("native"), formatters.TTY16m, nil)
 		if err != nil {
 			t.Errorf("Error opening file <%s>: %s", file, err.Error())
 			continue
@@ -221,7 +221,7 @@ func testHighlightingLineCount(t *testing.T, filenameWithPath string) {
 	}
 
 	// Then load the same file using one of our Readers
-	reader, err := NewReaderFromFilename(filenameWithPath, *styles.Get("native"), formatters.TTY16m)
+	reader, err := NewReaderFromFilename(filenameWithPath, *styles.Get("native"), formatters.TTY16m, nil)
 	if err != nil {
 		panic(err)
 	}
@@ -236,7 +236,7 @@ func testHighlightingLineCount(t *testing.T, filenameWithPath string) {
 
 func TestGetLongLine(t *testing.T) {
 	file := "../sample-files/very-long-line.txt"
-	reader, err := NewReaderFromFilename(file, *styles.Get("native"), formatters.TTY16m)
+	reader, err := NewReaderFromFilename(file, *styles.Get("native"), formatters.TTY16m, nil)
 	if err != nil {
 		panic(err)
 	}
@@ -286,7 +286,7 @@ func TestStatusText(t *testing.T) {
 	testStatusText(t, 1, 1, 1, "1 line  100%")
 
 	// Test with filename
-	testMe, err := NewReaderFromFilename(getSamplesDir()+"/empty", *styles.Get("native"), formatters.TTY16m)
+	testMe, err := NewReaderFromFilename(getSamplesDir()+"/empty", *styles.Get("native"), formatters.TTY16m, nil)
 	if err != nil {
 		panic(err)
 	}
@@ -301,7 +301,7 @@ func TestStatusText(t *testing.T) {
 
 func testCompressedFile(t *testing.T, filename string) {
 	filenameWithPath := getSamplesDir() + "/" + filename
-	reader, e := NewReaderFromFilename(filenameWithPath, *styles.Get("native"), formatters.TTY16m)
+	reader, e := NewReaderFromFilename(filenameWithPath, *styles.Get("native"), formatters.TTY16m, nil)
 	if e != nil {
 		t.Errorf("Error opening file <%s>: %s", filenameWithPath, e.Error())
 		panic(e)
@@ -374,7 +374,7 @@ func BenchmarkReaderDone(b *testing.B) {
 	b.ResetTimer()
 	for n := 0; n < b.N; n++ {
 		// This is our longest .go file
-		readMe, err := NewReaderFromFilename(filename, *styles.Get("native"), formatters.TTY16m)
+		readMe, err := NewReaderFromFilename(filename, *styles.Get("native"), formatters.TTY16m, nil)
 		if err != nil {
 			panic(err)
 		}
@@ -424,7 +424,7 @@ func BenchmarkReadLargeFile(b *testing.B) {
 
 	b.ResetTimer()
 	for n := 0; n < b.N; n++ {
-		readMe, err := NewReaderFromFilename(largeFileName, *styles.Get("native"), formatters.TTY16m)
+		readMe, err := NewReaderFromFilename(largeFileName, *styles.Get("native"), formatters.TTY16m, nil)
 		if err != nil {
 			panic(err)
 		}

moar.1

@@ -45,6 +45,12 @@ Print debug logs after exiting, less verbose than
 Scrolls automatically to follow piped input, just like
 .B tail \-f
 .TP
+\fB\-\-lang\fR=string
+Used for highlighting.
+Without this flag highlighting is based on the input file name.
+Valid values are MIME types like \fBtext/x-markdown\fP, file extensions like \fBmd\fP or language names like \fBmarkdown\fP.
+For the source of truth on what is supported exactly, look in https://github.com/alecthomas/chroma/tree/master/lexers/embedded or its parent directory.
+.TP
 \fB\-\-mousemode\fR={\fBauto\fR | \fBselect\fR | \fBscroll\fR}
 Guarantee selecting text with the mouse works but maybe not mouse scrolling.
 Or guarantee mouse scrolling works but selecting text requiring extra effort.

moar.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/alecthomas/chroma/v2"
 	"github.com/alecthomas/chroma/v2/formatters"
+	"github.com/alecthomas/chroma/v2/lexers"
 	"github.com/alecthomas/chroma/v2/styles"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/term"
@@ -156,6 +157,23 @@ func printProblemsHeader() {
 	fmt.Fprintln(os.Stderr, "NumCPU  :", runtime.NumCPU())
 }
 
+func parseLexerOption(lexerOption string) (chroma.Lexer, error) {
+	byMimeType := lexers.MatchMimeType(lexerOption)
+	if byMimeType != nil {
+		return byMimeType, nil
+	}
+
+	// Use Chroma's built-in fuzzy lexer picker
+	lexer := lexers.Get(lexerOption)
+	if lexer != nil {
+		return lexer, nil
+	}
+
+	return nil, fmt.Errorf(
+		"Look here for inspiration: https://github.com/alecthomas/chroma/tree/master/lexers/embedded",
+	)
+}
+
 func parseStyleOption(styleOption string) (chroma.Style, error) {
 	style, ok := styles.Registry[styleOption]
 	if !ok {
@@ -201,7 +219,7 @@ func parseStatusBarStyle(styleOption string) (m.StatusBarOption, error) {
 		return m.STATUSBAR_STYLE_BOLD, nil
 	}
 
-	return 0, fmt.Errorf("good ones are inverse, plain and bold")
+	return 0, fmt.Errorf("Good ones are inverse, plain and bold")
 }
 
 func parseUnprintableStyle(styleOption string) (m.UnprintableStyle, error) {
@@ -233,7 +251,7 @@ func parseShiftAmount(shiftAmount string) (uint, error) {
 	}
 
 	if value < 1 {
-		return 0, fmt.Errorf("Shift amount must be at least 1, was %d", value)
+		return 0, fmt.Errorf("Shift amount must be at least 1")
 	}
 
 	// Let's add an upper bound as well if / when requested
@@ -381,6 +399,9 @@ func main() {
 	style := flagSetFunc(flagSet,
 		"style", *styles.Registry["native"],
 		"Highlighting style from https://xyproto.github.io/splash/docs/longer/all.html", parseStyleOption)
+	lexer := flagSetFunc(flagSet,
+		"lang", nil,
+		"File contents, used for highlighting. Mime type or file extension (\"html\"). Default is to guess by filename.", parseLexerOption)
 
 	defaultFormatter, err := parseColorsOption("auto")
 	if err != nil {
@@ -527,7 +548,7 @@ func main() {
 		reader = m.NewReaderFromStream("", os.Stdin)
 	} else {
 		// Display the input file contents
-		reader, err = m.NewReaderFromFilename(*inputFilename, *style, formatter)
+		reader, err = m.NewReaderFromFilename(*inputFilename, *style, formatter, *lexer)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
 			os.Exit(1)