ci: add Go 1.11 to build matrix

.docker_env_file

@@ -1,6 +1,7 @@
 GH_USER
 GH_TOKEN
 GO_VERSION
+GO111MODULE
 CHROMEDRIVER_VERSION
 CHROME_CHANNEL
 TRAVIS

.gitignore

@@ -1,2 +1,2 @@
 /.bin
-_scripts/.vbash*
+.vbash*

.travis.yml

@@ -11,6 +11,7 @@ notifications:
 matrix:
   include:
     - env: CHROMEDRIVER_VERSION=2.40 CHROME_CHANNEL=beta GO_VERSION=go1.10.3
+    - env: CHROMEDRIVER_VERSION=2.40 CHROME_CHANNEL=beta GO_VERSION=12d0a2884a9e1a12050807393fe0266d1b0b40fd GO111MODULE=on
 
 services:
   - docker

_scripts/commonEnv.bash

@@ -20,6 +20,8 @@ then
 	export PROTOBUF_INSTALL_DIR=$CI_DEPENDENCIES_DIR/protobuf
 
 	export CHROMEDRIVER_INSTALL_DIR=$CI_DEPENDENCIES_DIR/chromedriver
+
+	export GOPROXY=file://$HOME/cachex
 fi
 
 if [ "${PROTOBUF_VERSION:-}" == "" ]

_scripts/commonFunctions.bash

@@ -177,5 +177,19 @@ install_deps()
 }
 export -f install_deps
 
+verifyGoGet()
+{
+	local pkg=$1
+	echo "Verifying go get for $pkg"
+	(
+	cd `mktemp -d`
+	export GOPATH=$PWD
+	GO111MODULE=auto
+	go env
+	go get $pkg
+	)
+}
+export -f verifyGoGet
+
 # **********************
 LOADED_COMMON_FUNCTIONS=true

_scripts/run_tests.sh

@@ -38,34 +38,27 @@ done
 #
 # run_nested_tests
 
-go generate $(subpackages)
+for i in $(find !(_vendor) -name go.mod -execdir pwd \;)
+do
+	pushd $i > /dev/null
+
+	go generate $(subpackages)
 
-ensure_go_formatted $(sub_git_files | grep -v '^_vendor/' | non_gen_go_files)
-ensure_go_gen_formatted $(sub_git_files | grep -v '^_vendor/' | gen_go_files)
+	ensure_go_formatted $(sub_git_files | grep -v '^_vendor/' | non_gen_go_files)
+	ensure_go_gen_formatted $(sub_git_files | grep -v '^_vendor/' | gen_go_files)
 
-go test $(subpackages)
+	go test $(subpackages)
 
-install_main_go $(subpackages)
+	install_main_go $(subpackages)
 
-# TODO remove once we have Go 1.11
-install_deps $(subpackages)
+	go vet $(subpackages)
 
-go vet $(subpackages)
+	popd > /dev/null
+done
 
 _scripts/update_readmes.sh
 
 if [ $(running_on_ci_server) == "yes" ]
 then
-	function verifyGoGet()
-	{
-		local pkg=$1
-		echo "Verifying go get for $pkg"
-		(
-		cd `mktemp -d`
-		export GOPATH=$PWD
-		go get $pkg
-		)
-	}
-
-	verifyGoGet "myitcv.io/cmd/concsh"
+	verifyGoGet myitcv.io/cmd/concsh
 fi

_scripts/setupCIEnv.sh

@@ -14,3 +14,8 @@ mkdir -p $PROTOBUF_INSTALL_DIR
 
 "${BASH_SOURCE%/*}/install_protobuf.sh"
 "${BASH_SOURCE%/*}/install_chromedriver.sh"
+
+mkdir -p $GOBIN
+
+cd $HOME
+git clone --depth=1 https://github.com/myitcv/cachex

_scripts/setupGOPATH.bash

@@ -13,6 +13,8 @@ fi
 
 autostash_or_export GOBIN="$(readlink -m "${BASH_SOURCE%/*}/../.bin")"
 
+autostash_or_export PATH="$GOBIN:$PATH"
+
 # Pre Go 1.11 check
 # [[ "$(go version | cut -d ' ' -f 3)" =~ go1.(9|10).[0-9]+ ]]
 

cmd/compressHistory/main.go

@@ -4,10 +4,11 @@
 package main
 
 import (
+	"bufio"
+	"errors"
 	"fmt"
+	"io"
 	"os"
-
-	"github.com/rogpeppe/rog-go/reverse"
 )
 
 // assumes that files have been provided in the order oldest to newset
@@ -28,7 +29,7 @@ func main() {
 		if err != nil {
 			panic(fmt.Sprintf("Could not open %v", fn))
 		}
-		scanner := reverse.NewScanner(fh)
+		scanner := NewScanner(fh)
 		for scanner.Scan() {
 			if _, ok := seen[scanner.Text()]; !ok {
 				seen[scanner.Text()] = struct{}{}
@@ -45,3 +46,210 @@ func main() {
 		fmt.Println(out[i])
 	}
 }
+
+// what follows is a copy of https://github.com/rogpeppe/rog-go/blob/master/reverse/scan.go
+
+const maxBufSize = 64 * 1024
+
+// Scanner presents the same interface as bufio.Scanner
+// except it scans tokens in reverse from the end of
+// a file instead of forwards from the beginning.
+//
+// It may not work correctly if the split function can return an
+// error from reading half way through a token.
+// That is not the case for any of the Scan* functions
+// in bufio.
+type Scanner struct {
+	r io.ReadSeeker
+
+	split bufio.SplitFunc
+
+	// buf holds currently buffered data.
+	buf []byte
+
+	// offset holds the file offset of the data
+	// in buf.
+	offset int64
+
+	// atEOF reports whether the buffer
+	// is located at the end of the file.
+	atEOF bool
+
+	// tokens holds any currently unscanned
+	// tokens in buf.
+	tokens [][]byte
+
+	// partialToken holds the size of the partial
+	// token at the start of buf.
+	partialToken int
+
+	// err holds any error encountered.
+	err error
+}
+
+// TODO make NewScanner take a ReaderAt rather
+// than a ReadSeeker.
+
+// TODO provide a FileOffset method that returns
+// the offset of the token in the file.
+
+// NewScanner returns a new Scanner to read tokens
+// in reverse from r. The split function defaults to bufio.ScanLines.
+func NewScanner(r io.ReadSeeker) *Scanner {
+	b := &Scanner{
+		r:     r,
+		buf:   make([]byte, 4096),
+		atEOF: true,
+		split: bufio.ScanLines,
+	}
+	b.offset, b.err = r.Seek(0, 2)
+	return b
+}
+
+func (b *Scanner) fillbuf() error {
+	b.tokens = b.tokens[:0]
+	if b.offset == 0 {
+		return io.EOF
+	}
+	space := len(b.buf) - b.partialToken
+	if space == 0 {
+		// Partial token fills the buffer, so expand it.
+		if len(b.buf) >= maxBufSize {
+			return errors.New("token too long")
+		}
+		n := len(b.buf) * 2
+		if n > maxBufSize {
+			n = maxBufSize
+		}
+		newBuf := make([]byte, n)
+		copy(newBuf, b.buf[0:b.partialToken])
+		b.buf = newBuf
+		space = len(b.buf) - b.partialToken
+	}
+	if int64(space) > b.offset {
+		// We have less than the given buffer's space
+		// left to read, so shrink the buffer to simplify
+		// the remaining logic by preserving the
+		// invariants that b.partialToken + space == len(buf)
+		// and the data is read into the start of buf.
+		b.buf = b.buf[0 : b.partialToken+int(b.offset)]
+		space = len(b.buf) - b.partialToken
+	}
+	newOffset := b.offset - int64(space)
+	// Copy old partial token to end of buffer.
+	copy(b.buf[space:], b.buf[0:b.partialToken])
+	_, err := b.r.Seek(newOffset, 0)
+	if err != nil {
+		return err
+	}
+	b.offset = newOffset
+	if _, err := io.ReadFull(b.r, b.buf[0:space]); err != nil {
+		// TODO Cope better with io.UnexpectedEOF at the
+		// end of a file - historically some versions of NFS
+		// can report a file size that's larger than the actual
+		// file size.
+		return err
+	}
+	// Populate b.tokens with the tokens in the buffer.
+	if b.offset > 0 {
+		// We're not at the start of the file - read the first
+		// token to find out where the token boundary is, but
+		// don't treat it as an actual token, because we're
+		// probably not at its start.
+		advance, _, err := b.split(b.buf, b.atEOF)
+		if err != nil {
+			// If the split function can return an error
+			// when starting at a non-token boundary, this
+			// will happen and there's not much we can do
+			// about it other than scanning forward a byte
+			// at a time in a horribly inefficient manner,
+			// so just return the error.
+			return err
+		}
+		b.partialToken = advance
+		if advance == 0 || advance == len(b.buf) {
+			// There are no more tokens in the buffer,
+			// or the single token fills the entire buffer
+			// so try again (the buffer will expand if
+			// necessary)
+			return b.fillbuf()
+		}
+	} else {
+		b.partialToken = 0
+	}
+	for i := b.partialToken; i < len(b.buf); {
+		advance, token, err := b.split(b.buf[i:], b.atEOF)
+		if err != nil {
+			// We've got an error - avoid returning
+			// any tokens encountered earlier in this
+			// buffer scan, otherwise we risk skipping
+			// tokens before returning the error.
+			b.tokens = b.tokens[:0]
+			return err
+		}
+		if advance == 0 {
+			// There's no remaining token in the buffer.
+			break
+		}
+		b.tokens = append(b.tokens, token)
+		i += advance
+	}
+	b.atEOF = false
+	if len(b.tokens) == 0 {
+		// There were no tokens found - that means that
+		// although we did find a partial token at the start,
+		// there were no other tokens, so we need
+		// to scan back further.
+		return b.fillbuf()
+	}
+	return nil
+}
+
+// Scan advances the Scanner to the next token, which will then be
+// available through the Bytes or Text method. It returns false when the
+// scan stops, either by reaching the end of the input or an error.
+// After Scan returns false, the Err method will return any error that
+// occurred during scanning, except that if it was io.EOF, Err
+// will return nil.
+func (b *Scanner) Scan() bool {
+	if len(b.tokens) > 0 {
+		b.tokens = b.tokens[0 : len(b.tokens)-1]
+	}
+	if len(b.tokens) > 0 {
+		return true
+	}
+	if b.err != nil {
+		return false
+	}
+	b.err = b.fillbuf()
+	return len(b.tokens) > 0
+}
+
+// Split sets the split function for the Scanner. If called, it must be
+// called before Scan. The default split function is bufio.ScanLines.
+func (b *Scanner) Split(split bufio.SplitFunc) {
+	b.split = split
+}
+
+// Bytes returns the most recent token generated by a call to Scan.
+// The underlying array may point to data that will be overwritten
+// by a subsequent call to Scan. It does no allocation.
+func (b *Scanner) Bytes() []byte {
+	return b.tokens[len(b.tokens)-1]
+}
+
+// Text returns the most recent token generated by a call to Scan
+// as a newly allocated string holding its bytes.
+func (b *Scanner) Text() string {
+	return string(b.Bytes())
+}
+
+func (b *Scanner) Err() error {
+	if len(b.tokens) > 0 {
+		return nil
+	}
+	if b.err == io.EOF {
+		return nil
+	}
+	return b.err
+}

go.mod

@@ -0,0 +1,32 @@
+module myitcv.io
+
+require (
+	github.com/golang/protobuf v1.1.0
+	github.com/gopherjs/gopherjs v0.0.0-20180628210949-0892b62f0d9f
+	github.com/gopherjs/jsbuiltin v0.0.0-20180426082241-50091555e127
+	github.com/hpcloud/tail v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jteeuwen/go-bindata v3.0.7+incompatible
+	github.com/kisielk/gotool v1.0.0
+	github.com/kr/fs v0.1.0
+	github.com/kr/pretty v0.1.0 // indirect
+	github.com/onsi/ginkgo v1.6.0 // indirect
+	github.com/onsi/gomega v1.4.1 // indirect
+	github.com/russross/blackfriday v1.5.1
+	github.com/sclevine/agouti v3.0.0+incompatible
+	golang.org/x/net v0.0.0-20180731172858-49c15d80dfbc
+	golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f // indirect
+	golang.org/x/text v0.3.0 // indirect
+	golang.org/x/tools v0.0.0-20180803180156-3c07937fe18c
+	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127
+	gopkg.in/fsnotify.v1 v1.4.7 // indirect
+	gopkg.in/fsnotify/fsnotify.v1 v1.4.7
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	gopkg.in/yaml.v2 v2.2.1 // indirect
+	honnef.co/go/js/dom v0.0.0-20180323154144-6da835bec70f
+	honnef.co/go/js/util v0.0.0-20150216223935-96b8dd9d1621 // indirect
+	honnef.co/go/js/xhr v0.0.0-20150307031022-00e3346113ae
+	mvdan.cc/sh v2.5.0+incompatible
+)
+
+replace github.com/gopherjs/gopherjs => github.com/myitcv/gopherjs v0.0.0-20180810213059-af1a9351d8ec