Skip to content

Commit

Permalink
Merge branch 'master' into windowexec
Browse files Browse the repository at this point in the history
  • Loading branch information
zz-jason committed Jan 14, 2019
2 parents 7db14b1 + 4b98ad6 commit b6673b9
Show file tree
Hide file tree
Showing 153 changed files with 4,527 additions and 1,312 deletions.
2 changes: 1 addition & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ A Contributor refers to the person who contributes to the following projects:

## How to become a TiDB Contributor?

If a PR (Pull Request) submitted to the TiDB / TiKV / TiSpark / PD / DocsDocs-cn projects by you is approved and merged, then you become a TiDB Contributor.
If a PR (Pull Request) submitted to the TiDB/TiKV/TiSpark/PD/Docs/Docs-cn projects by you is approved and merged, then you become a TiDB Contributor.

You are also encouraged to participate in the projects in the following ways:
- Actively answer technical questions asked by community users.
Expand Down
45 changes: 19 additions & 26 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ path_to_add := $(addsuffix /bin,$(subst :,/bin:,$(GOPATH)))
export PATH := $(path_to_add):$(PATH)

GO := GO111MODULE=on go
GOBUILD := CGO_ENABLED=0 $(GO) build $(BUILD_FLAG)
GOBUILD := CGO_ENABLED=1 $(GO) build $(BUILD_FLAG)
GOTEST := CGO_ENABLED=1 $(GO) test -p 3
OVERALLS := CGO_ENABLED=1 GO111MODULE=on overalls

Expand All @@ -24,8 +24,8 @@ PACKAGES := $$($(PACKAGE_LIST))
PACKAGE_DIRECTORIES := $(PACKAGE_LIST) | sed 's|github.com/pingcap/$(PROJECT)/||'
FILES := $$(find $$($(PACKAGE_DIRECTORIES)) -name "*.go")

GOFAIL_ENABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|tools)" | xargs gofail enable)
GOFAIL_DISABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|tools)" | xargs gofail disable)
GOFAIL_ENABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|tools)" | xargs tools/bin/gofail enable)
GOFAIL_DISABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|tools)" | xargs tools/bin/gofail disable)

LDFLAGS += -X "github.com/pingcap/parser/mysql.TiDBReleaseVersion=$(shell git describe --tags --dirty)"
LDFLAGS += -X "github.com/pingcap/tidb/util/printer.TiDBBuildTS=$(shell date -u '+%Y-%m-%d %I:%M:%S')"
Expand Down Expand Up @@ -123,11 +123,7 @@ ifeq ("$(TRAVIS_COVERAGE)", "1")
bash <(curl -s https://codecov.io/bash)
endif

gotest:
@rm -rf $GOPATH/bin/gofail
$(GO) get github.com/pingcap/gofail
@which gofail
@$(GOFAIL_ENABLE)
gotest: gofail-enable
ifeq ("$(TRAVIS_COVERAGE)", "1")
@echo "Running in TRAVIS_COVERAGE mode."
@export log_level=error; \
Expand All @@ -140,23 +136,17 @@ else
endif
@$(GOFAIL_DISABLE)

race:
$(GO) get github.com/pingcap/gofail
@$(GOFAIL_ENABLE)
race: gofail-enable
@export log_level=debug; \
$(GOTEST) -timeout 20m -race $(PACKAGES) || { $(GOFAIL_DISABLE); exit 1; }
@$(GOFAIL_DISABLE)

leak:
$(GO) get github.com/pingcap/gofail
@$(GOFAIL_ENABLE)
leak: gofail-enable
@export log_level=debug; \
$(GOTEST) -tags leak $(PACKAGES) || { $(GOFAIL_DISABLE); exit 1; }
@$(GOFAIL_DISABLE)

tikv_integration_test:
$(GO) get github.com/pingcap/gofail
@$(GOFAIL_ENABLE)
tikv_integration_test: gofail-enable
$(GOTEST) ./store/tikv/. -with-tikv=true || { $(GOFAIL_DISABLE); exit 1; }
@$(GOFAIL_DISABLE)

Expand Down Expand Up @@ -200,37 +190,40 @@ importer:
checklist:
cat checklist.md

gofail-enable:
gofail-enable: tools/bin/gofail
# Converting gofail failpoints...
@$(GOFAIL_ENABLE)

gofail-disable:
gofail-disable: tools/bin/gofail
# Restoring gofail failpoints...
@$(GOFAIL_DISABLE)

checkdep:
$(GO) list -f '{{ join .Imports "\n" }}' github.com/pingcap/tidb/store/tikv | grep ^github.com/pingcap/parser$$ || exit 0; exit 1

tools/bin/megacheck:
tools/bin/megacheck: tools/check/go.mod
cd tools/check; \
$go build -o ../bin/megacheck honnef.co/go/tools/cmd/megacheck
$(GO) build -o ../bin/megacheck honnef.co/go/tools/cmd/megacheck

tools/bin/revive:
tools/bin/revive: tools/check/go.mod
cd tools/check; \
$(GO) build -o ../bin/revive github.com/mgechev/revive

tools/bin/goword:
tools/bin/goword: tools/check/go.mod
cd tools/check; \
$(GO) build -o ../bin/goword github.com/chzchzchz/goword

tools/bin/gometalinter:
tools/bin/gometalinter: tools/check/go.mod
cd tools/check; \
$(GO) build -o ../bin/gometalinter gopkg.in/alecthomas/gometalinter.v2

tools/bin/gosec:
tools/bin/gosec: tools/check/go.mod
cd tools/check; \
$(GO) build -o ../bin/gosec github.com/securego/gosec/cmd/gosec

tools/bin/errcheck:
tools/bin/errcheck: tools/check/go.mod
cd tools/check; \
$(GO) build -o ../bin/errcheck github.com/kisielk/errcheck

tools/bin/gofail: go.mod
$(GO) build -o $@ github.com/pingcap/gofail
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

[![Build Status](https://travis-ci.org/pingcap/tidb.svg?branch=master)](https://travis-ci.org/pingcap/tidb)
[![Go Report Card](https://goreportcard.com/badge/github.com/pingcap/tidb)](https://goreportcard.com/report/github.com/pingcap/tidb)
![GitHub release](https://img.shields.io/github/release/pingcap/tidb.svg)
![GitHub release](https://img.shields.io/github/tag/pingcap/tidb.svg?label=release)
[![CircleCI Status](https://circleci.com/gh/pingcap/tidb.svg?style=shield)](https://circleci.com/gh/pingcap/tidb)
[![Coverage Status](https://codecov.io/gh/pingcap/tidb/branch/master/graph/badge.svg)](https://codecov.io/gh/pingcap/tidb)

Expand Down
6 changes: 3 additions & 3 deletions cmd/benchdb/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,13 +118,13 @@ func (ut *benchDB) mustExec(sql string) {
if len(rss) > 0 {
ctx := context.Background()
rs := rss[0]
chk := rs.NewChunk()
req := rs.NewRecordBatch()
for {
err := rs.Next(ctx, chk)
err := rs.Next(ctx, req)
if err != nil {
log.Fatal(err)
}
if chk.NumRows() == 0 {
if req.NumRows() == 0 {
break
}
}
Expand Down
159 changes: 159 additions & 0 deletions cmd/pluginpkg/pluginpkg.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
// Copyright 2018 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
"context"
"encoding/json"
"flag"
"fmt"
"log"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"text/template"
"time"

"github.com/BurntSushi/toml"
)

var (
pkgDir string
outDir string
)

const codeTemplate = `
package main
import (
"github.com/pingcap/tidb/plugin"
"github.com/pingcap/tidb/sessionctx/variable"
)
func PluginManifest() *plugin.Manifest {
return plugin.ExportManifest(&plugin.{{.kind}}Manifest{
Manifest: plugin.Manifest{
Kind: plugin.{{.kind}},
Name: "{{.name}}",
Description: "{{.description}}",
Version: {{.version}},
RequireVersion: map[string]uint16{},
License: "{{.license}}",
BuildTime: "{{.buildTime}}",
SysVars: map[string]*variable.SysVar{
{{range .sysVars}}
"{{.name}}": {
Scope: variable.Scope{{.scope}},
Name: "{{.name}}",
Value: "{{.value}}",
},
{{end}}
},
Validate: {{.validate}},
OnInit: {{.onInit}},
OnShutdown: {{.onShutdown}},
},
{{range .export}}
{{.extPoint}}: {{.impl}},
{{end}}
})
}
`

func init() {
flag.StringVar(&pkgDir, "pkg-dir", "", "plugin package folder path")
flag.StringVar(&outDir, "out-dir", "", "plugin packaged folder path")
flag.Usage = usage
}

func usage() {
log.Printf("Usage: %s --pkg-dir [plugin source pkg folder] --outDir-dir [outDir-dir]\n", path.Base(os.Args[0]))
flag.PrintDefaults()
os.Exit(1)
}

func main() {
flag.Parse()
if pkgDir == "" || outDir == "" {
flag.Usage()
}
var manifest map[string]interface{}
_, err := toml.DecodeFile(filepath.Join(pkgDir, "manifest.toml"), &manifest)
if err != nil {
log.Printf("read pkg %s's manifest failure, %+v\n", pkgDir, err)
os.Exit(1)
}
manifest["buildTime"] = time.Now().String()

pluginName := manifest["name"].(string)
if strings.Contains(pluginName, "-") {
log.Printf("plugin name should not contain '-'\n")
os.Exit(1)
}
if pluginName != filepath.Base(pkgDir) {
log.Printf("plugin package must be same with plugin name in manifest file\n")
os.Exit(1)
}

version := manifest["version"].(string)
tmpl, err := template.New("gen-plugin").Parse(codeTemplate)
if err != nil {
log.Printf("generate code failure during parse template, %+v\n", err)
os.Exit(1)
}

genFileName := filepath.Join(pkgDir, filepath.Base(pkgDir)+".gen.go")
genFile, err := os.OpenFile(genFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
if err != nil {
log.Printf("generate code failure during prepare output file, %+v\n", err)
os.Exit(1)
}
defer func() {
err1 := os.Remove(genFileName)
if err1 != nil {
log.Printf("remove tmp file %s failure, please clean up manually at %v", genFileName, err1)
}
}()

err = tmpl.Execute(genFile, manifest)
if err != nil {
log.Printf("generate code failure during generating code, %+v\n", err)
os.Exit(1)
}

outputFile := filepath.Join(outDir, pluginName+"-"+version+".so")
pluginPath := `-pluginpath=` + pluginName + "-" + version
ctx := context.Background()
buildCmd := exec.CommandContext(ctx, "go", "build",
"-ldflags", pluginPath,
"-buildmode=plugin",
"-o", outputFile, pkgDir)
buildCmd.Stderr = os.Stderr
buildCmd.Stdout = os.Stdout
buildCmd.Env = append(os.Environ(), "GO111MODULE=on")
err = buildCmd.Run()
if err != nil {
log.Printf("compile plugin source code failure, %+v\n", err)
os.Exit(1)
}
fmt.Printf(`Package "%s" as plugin "%s" success.`+"\nManifest:\n", pkgDir, outputFile)
encoder := json.NewEncoder(os.Stdout)
encoder.SetIndent(" ", "\t")
err = encoder.Encode(manifest)
if err != nil {
log.Printf("print manifest detail failure, err: %v", err)
}
}
25 changes: 24 additions & 1 deletion config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ type Config struct {
TiKVClient TiKVClient `toml:"tikv-client" json:"tikv-client"`
Binlog Binlog `toml:"binlog" json:"binlog"`
CompatibleKillQuery bool `toml:"compatible-kill-query" json:"compatible-kill-query"`
Plugin Plugin `toml:"plugin" json:"plugin"`
}

// Log is the log section of config.
Expand Down Expand Up @@ -240,8 +241,18 @@ type TiKVClient struct {
GrpcKeepAliveTimeout uint `toml:"grpc-keepalive-timeout" json:"grpc-keepalive-timeout"`
// CommitTimeout is the max time which command 'commit' will wait.
CommitTimeout string `toml:"commit-timeout" json:"commit-timeout"`

// MaxTxnTimeUse is the max time a Txn may use (in seconds) from its startTS to commitTS.
MaxTxnTimeUse uint `toml:"max-txn-time-use" json:"max-txn-time-use"`

// MaxBatchSize is the max batch size when calling batch commands API.
MaxBatchSize uint `toml:"max-batch-size" json:"max-batch-size"`
// If TiKV load is greater than this, TiDB will wait for a while to avoid little batch.
OverloadThreshold uint `toml:"overload-threshold" json:"overload-threshold"`
// MaxBatchWaitTime in nanosecond is the max wait time for batch.
MaxBatchWaitTime time.Duration `toml:"max-batch-wait-time" json:"max-batch-wait-time"`
// BatchWaitSize is the max wait size for batch.
BatchWaitSize uint `toml:"batch-wait-size" json:"batch-wait-size"`
}

// Binlog is the config for binlog.
Expand All @@ -255,6 +266,12 @@ type Binlog struct {
BinlogSocket string `toml:"binlog-socket" json:"binlog-socket"`
}

// Plugin is the config for plugin
type Plugin struct {
Dir string `toml:"dir" json:"dir"`
Load string `toml:"load" json:"load"`
}

var defaultConf = Config{
Host: "0.0.0.0",
AdvertiseAddress: "",
Expand Down Expand Up @@ -328,7 +345,13 @@ var defaultConf = Config{
GrpcKeepAliveTime: 10,
GrpcKeepAliveTimeout: 3,
CommitTimeout: "41s",
MaxTxnTimeUse: 590,

MaxTxnTimeUse: 590,

MaxBatchSize: 128,
OverloadThreshold: 200,
MaxBatchWaitTime: 0,
BatchWaitSize: 8,
},
Binlog: Binlog{
WriteTimeout: "15s",
Expand Down
11 changes: 10 additions & 1 deletion config/config.toml.example
Original file line number Diff line number Diff line change
Expand Up @@ -231,14 +231,23 @@ grpc-keepalive-time = 10
# and if no activity is seen even after that the connection is closed.
grpc-keepalive-timeout = 3

# max time for commit command, must be twice bigger than raft election timeout.
# Max time for commit command, must be twice bigger than raft election timeout.
commit-timeout = "41s"

# The max time a Txn may use (in seconds) from its startTS to commitTS.
# We use it to guarantee GC worker will not influence any active txn. Please make sure that this
# value is less than gc_life_time - 10s.
max-txn-time-use = 590

# Max batch size in gRPC.
max-batch-size = 128
# Overload threshold of TiKV.
overload-threshold = 200
# Max batch wait time in nanosecond to avoid waiting too long. 0 means disable this feature.
max-batch-wait-time = 0
# Batch wait size, to avoid waiting too long.
batch-wait-size = 8

[txn-local-latches]
# Enable local latches for transactions. Enable it when
# there are lots of conflicts between transactions.
Expand Down
Loading

0 comments on commit b6673b9

Please sign in to comment.