From 454da027b16284767a27e90431fdb4bf617840ce Mon Sep 17 00:00:00 2001 From: Adrien Petel Date: Wed, 30 Aug 2017 15:55:17 +0400 Subject: [PATCH 01/38] add DropAllIndexes() method (#25) Create a new method to drop all the indexes of a collection in a single call --- session.go | 36 +++++++++++++++++++++++------------- session_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/session.go b/session.go index 6827dffe5..d18277be4 100644 --- a/session.go +++ b/session.go @@ -1499,6 +1499,29 @@ func (c *Collection) DropIndexName(name string) error { return nil } +// DropAllIndexes drops all the indexes from the c collection +func (c *Collection) DropAllIndexes() error { + session := c.Database.Session + session.ResetIndexCache() + + session = session.Clone() + defer session.Close() + + db := c.Database.With(session) + result := struct { + ErrMsg string + Ok bool + }{} + err := db.Run(bson.D{{"dropIndexes", c.Name}, {"index", "*"}}, &result) + if err != nil { + return err + } + if !result.Ok { + return errors.New(result.ErrMsg) + } + return nil +} + // nonEventual returns a clone of session and ensures it is not Eventual. // This guarantees that the server that is used for queries may be reused // afterwards when a cursor is received. @@ -1512,19 +1535,6 @@ func (session *Session) nonEventual() *Session { // Indexes returns a list of all indexes for the collection. // -// For example, this snippet would drop all available indexes: -// -// indexes, err := collection.Indexes() -// if err != nil { -// return err -// } -// for _, index := range indexes { -// err = collection.DropIndex(index.Key...) -// if err != nil { -// return err -// } -// } -// // See the EnsureIndex method for more details on indexes. func (c *Collection) Indexes() (indexes []Index, err error) { cloned := c.Database.Session.nonEventual() diff --git a/session_test.go b/session_test.go index 912f1c92a..e29221cb4 100644 --- a/session_test.go +++ b/session_test.go @@ -3478,6 +3478,31 @@ func (s *S) TestEnsureIndexDropIndexName(c *C) { c.Assert(err, ErrorMatches, "index not found.*") } +func (s *S) TestEnsureIndexDropAllIndexes(c *C) { + session, err := mgo.Dial("localhost:40001") + c.Assert(err, IsNil) + defer session.Close() + + coll := session.DB("mydb").C("mycoll") + + err = coll.EnsureIndexKey("a") + c.Assert(err, IsNil) + + err = coll.EnsureIndexKey("b") + c.Assert(err, IsNil) + + err = coll.DropAllIndexes() + c.Assert(err, IsNil) + + sysidx := session.DB("mydb").C("system.indexes") + + err = sysidx.Find(M{"name": "a_1"}).One(nil) + c.Assert(err, Equals, mgo.ErrNotFound) + + err = sysidx.Find(M{"name": "b_1"}).One(nil) + c.Assert(err, Equals, mgo.ErrNotFound) +} + func (s *S) TestEnsureIndexCaching(c *C) { session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) From 93aaa6e1150dee0f70ac9aeaf3d7d945bd48c9cd Mon Sep 17 00:00:00 2001 From: Dom Date: Wed, 30 Aug 2017 17:40:17 +0100 Subject: [PATCH 02/38] readme: credit @feliixx for #25 (#26) --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 7c4a4191d..0bf24c430 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Further PR's (with tests) are welcome, but please maintain backwards compatibili * Improved multi-document transaction performance ([details](https://github.com/globalsign/mgo/pull/10), [more](https://github.com/globalsign/mgo/pull/11), [more](https://github.com/globalsign/mgo/pull/16)) * Fixes cursor timeouts ([detials](https://jira.mongodb.org/browse/SERVER-24899)) * Support index hints and timeouts for count queries ([details](https://github.com/globalsign/mgo/pull/17)) +* Allow dropping all indexes on a collection ([details](https://github.com/globalsign/mgo/pull/25)) --- @@ -30,6 +31,7 @@ Further PR's (with tests) are welcome, but please maintain backwards compatibili * @cezarsa * @drichelson * @eaglerayp +* @feliixx * @fmpwizard * @jameinel * @Reenjii From 165af6858035e90c6eba5dc98dd986586d90e57e Mon Sep 17 00:00:00 2001 From: Adrien Petel Date: Wed, 6 Sep 2017 13:23:32 +0400 Subject: [PATCH 03/38] send metadata during handshake (#28) fix [#484](https://github.com/go-mgo/mgo/issues/484) Annotate connections with metadata provided by the connecting client. informations send: { "aplication": { // optional "name": "myAppName" } "driver": { "name": "mgo", "version": "v2" }, "os": { "type": runtime.GOOS, "architecture": runtime.GOARCH } } to set "application.name", add `appname` param in options of string connection URI, for example : "mongodb://localhost:27017?appname=myAppName" --- cluster.go | 17 +++++++++++++++-- session.go | 16 +++++++++++++++- session_test.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/cluster.go b/cluster.go index d43245649..81e4f7ff5 100644 --- a/cluster.go +++ b/cluster.go @@ -30,6 +30,7 @@ import ( "errors" "fmt" "net" + "runtime" "strconv" "strings" "sync" @@ -61,9 +62,10 @@ type mongoCluster struct { cachedIndex map[string]bool sync chan bool dial dialer + appName string } -func newCluster(userSeeds []string, direct, failFast bool, dial dialer, setName string) *mongoCluster { +func newCluster(userSeeds []string, direct, failFast bool, dial dialer, setName string, appName string) *mongoCluster { cluster := &mongoCluster{ userSeeds: userSeeds, references: 1, @@ -71,6 +73,7 @@ func newCluster(userSeeds []string, direct, failFast bool, dial dialer, setName failFast: failFast, dial: dial, setName: setName, + appName: appName, } cluster.serverSynced.L = cluster.RWMutex.RLocker() cluster.sync = make(chan bool, 1) @@ -144,7 +147,17 @@ func (cluster *mongoCluster) isMaster(socket *mongoSocket, result *isMasterResul // Monotonic let's it talk to a slave and still hold the socket. session := newSession(Monotonic, cluster, 10*time.Second) session.setSocket(socket) - err := session.Run("ismaster", result) + + // provide some meta infos on the client, + // see https://github.com/mongodb/specifications/blob/master/source/mongodb-handshake/handshake.rst#connection-handshake + // for details + metaInfo := bson.M{"driver": bson.M{"name": "mgo", "version": "globalsign"}, + "os": bson.M{"type": runtime.GOOS, "architecture": runtime.GOARCH}} + + if cluster.appName != "" { + metaInfo["application"] = bson.M{"name": cluster.appName} + } + err := session.Run(bson.D{{"isMaster", 1}, {"client", metaInfo}}, result) session.Close() return err } diff --git a/session.go b/session.go index d18277be4..cfe271ff9 100644 --- a/session.go +++ b/session.go @@ -235,6 +235,10 @@ const ( // Defines the per-server socket pool limit. Defaults to 4096. // See Session.SetPoolLimit for details. // +// appName= +// +// The identifier of the client application which ran the operation. This +// param can't exceed 128 bytes // // Relevant documentation: // @@ -279,6 +283,7 @@ func ParseURL(url string) (*DialInfo, error) { source := "" setName := "" poolLimit := 0 + appName := "" readPreferenceMode := Primary var readPreferenceTagSets []bson.D for _, opt := range uinfo.options { @@ -296,6 +301,11 @@ func ParseURL(url string) (*DialInfo, error) { if err != nil { return nil, errors.New("bad value for maxPoolSize: " + opt.value) } + case "appName": + if len(opt.value) > 128 { + return nil, errors.New("appName too long, must be < 128 bytes: " + opt.value) + } + appName = opt.value case "readPreference": switch opt.value { case "nearest": @@ -350,6 +360,7 @@ func ParseURL(url string) (*DialInfo, error) { Service: service, Source: source, PoolLimit: poolLimit, + AppName: appName, ReadPreference: &ReadPreference{ Mode: readPreferenceMode, TagSets: readPreferenceTagSets, @@ -409,6 +420,9 @@ type DialInfo struct { // See Session.SetPoolLimit for details. PoolLimit int + // The identifier of the client application which ran the operation. + AppName string + // ReadPreference defines the manner in which servers are chosen. See // Session.SetMode and Session.SelectServers. ReadPreference *ReadPreference @@ -472,7 +486,7 @@ func DialWithInfo(info *DialInfo) (*Session, error) { } addrs[i] = addr } - cluster := newCluster(addrs, info.Direct, info.FailFast, dialer{info.Dial, info.DialServer}, info.ReplicaSetName) + cluster := newCluster(addrs, info.Direct, info.FailFast, dialer{info.Dial, info.DialServer}, info.ReplicaSetName, info.AppName) session := newSession(Eventual, cluster, info.Timeout) session.defaultdb = info.Database if session.defaultdb == "" { diff --git a/session_test.go b/session_test.go index e29221cb4..216474889 100644 --- a/session_test.go +++ b/session_test.go @@ -200,6 +200,50 @@ func (s *S) TestURLInvalidReadPreferenceTags(c *C) { } } +func (s *S) TestURLWithAppName(c *C) { + if !s.versionAtLeast(3, 4) { + c.Skip("appName depends on MongoDB 3.4+") + } + appName := "myAppName" + session, err := mgo.Dial("localhost:40001?appName=" + appName) + c.Assert(err, IsNil) + defer session.Close() + + db := session.DB("mydb") + + err = db.Run(bson.D{{"profile", 2}}, nil) + c.Assert(err, IsNil) + + coll := db.C("mycoll") + err = coll.Insert(M{"a": 1, "b": 2}) + c.Assert(err, IsNil) + + result := struct{ A, B int }{} + err = coll.Find(M{"a": 1}).One(&result) + c.Assert(err, IsNil) + + profileResult := struct { + AppName string `bson:"appName"` + }{} + + err = db.C("system.profile").Find(nil).Sort("-ts").One(&profileResult) + c.Assert(err, IsNil) + c.Assert(appName, Equals, profileResult.AppName) + // reset profiling to 0 as it add unecessary overhead to all other test + err = db.Run(bson.D{{"profile", 0}}, nil) + c.Assert(err, IsNil) +} + +func (s *S) TestURLWithAppNameTooLong(c *C) { + if !s.versionAtLeast(3, 4) { + c.Skip("appName depends on MongoDB 3.4+") + } + appName := "myAppNameWayTooLongmyAppNameWayTooLongmyAppNameWayTooLongmyAppNameWayTooLong" + appName += appName + _, err := mgo.Dial("localhost:40001?appName=" + appName) + c.Assert(err, ErrorMatches, "appName too long, must be < 128 bytes: "+appName) +} + func (s *S) TestInsertFindOne(c *C) { session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) From a76b1a0386e403ec1373d1d302bca8566f8bfa0f Mon Sep 17 00:00:00 2001 From: Dom Date: Wed, 6 Sep 2017 15:59:38 +0100 Subject: [PATCH 04/38] Update README to add appName (#32) * docs: elaborate on what appName does * readme: add appName to changes --- README.md | 3 ++- session.go | 14 +++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 0bf24c430..fce236bf7 100644 --- a/README.md +++ b/README.md @@ -19,9 +19,10 @@ Further PR's (with tests) are welcome, but please maintain backwards compatibili * Fixes timezone handling ([details](https://github.com/go-mgo/mgo/pull/464)) * Integration tests run against newest MongoDB 3.2 releases ([details](https://github.com/globalsign/mgo/pull/4), [more](https://github.com/globalsign/mgo/pull/24)) * Improved multi-document transaction performance ([details](https://github.com/globalsign/mgo/pull/10), [more](https://github.com/globalsign/mgo/pull/11), [more](https://github.com/globalsign/mgo/pull/16)) -* Fixes cursor timeouts ([detials](https://jira.mongodb.org/browse/SERVER-24899)) +* Fixes cursor timeouts ([details](https://jira.mongodb.org/browse/SERVER-24899)) * Support index hints and timeouts for count queries ([details](https://github.com/globalsign/mgo/pull/17)) * Allow dropping all indexes on a collection ([details](https://github.com/globalsign/mgo/pull/25)) +* Annotates log entries/profiler output with optional appName on 3.4+ ([details](https://github.com/globalsign/mgo/pull/28)) --- diff --git a/session.go b/session.go index cfe271ff9..65ea64bb3 100644 --- a/session.go +++ b/session.go @@ -157,9 +157,9 @@ const ( // topology. // // Dial will timeout after 10 seconds if a server isn't reached. The returned -// session will timeout operations after one minute by default if servers -// aren't available. To customize the timeout, see DialWithTimeout, -// SetSyncTimeout, and SetSocketTimeout. +// session will timeout operations after one minute by default if servers aren't +// available. To customize the timeout, see DialWithTimeout, SetSyncTimeout, and +// SetSocketTimeout. // // This method is generally called just once for a given cluster. Further // sessions to the same cluster are then established using the New or Copy @@ -184,8 +184,8 @@ const ( // If the port number is not provided for a server, it defaults to 27017. // // The username and password provided in the URL will be used to authenticate -// into the database named after the slash at the end of the host names, or -// into the "admin" database if none is provided. The authentication information +// into the database named after the slash at the end of the host names, or into +// the "admin" database if none is provided. The authentication information // will persist in sessions obtained through the New method as well. // // The following connection options are supported after the question mark: @@ -237,8 +237,8 @@ const ( // // appName= // -// The identifier of the client application which ran the operation. This -// param can't exceed 128 bytes +// The identifier of this client application. This parameter is used to +// annotate logs / profiler output and cannot exceed 128 bytes. // // Relevant documentation: // From 25200e4c5773d49b2aae11f2f0f8b06773003cbc Mon Sep 17 00:00:00 2001 From: Adrien Petel Date: Mon, 11 Sep 2017 13:35:18 +0400 Subject: [PATCH 05/38] add method CreateView() (#33) Fix #30. Thanks to @feliixx for the time and effort. --- session.go | 24 ++++++++++ session_test.go | 125 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+) diff --git a/session.go b/session.go index 65ea64bb3..b1bbcdbf7 100644 --- a/session.go +++ b/session.go @@ -666,6 +666,30 @@ func (db *Database) C(name string) *Collection { return &Collection{db, name, db.Name + "." + name} } +// CreateView creates a view as the result of the applying the specified +// aggregation pipeline to the source collection or view. Views act as +// read-only collections, and are computed on demand during read operations. +// MongoDB executes read operations on views as part of the underlying aggregation pipeline. +// +// For example: +// +// db := session.DB("mydb") +// db.CreateView("myview", "mycoll", []bson.M{{"$match": bson.M{"c": 1}}}, nil) +// view := db.C("myview") +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/core/views/ +// https://docs.mongodb.com/manual/reference/method/db.createView/ +// +func (db *Database) CreateView(view string, source string, pipeline interface{}, collation *Collation) error { + command := bson.D{{"create", view}, {"viewOn", source}, {"pipeline", pipeline}} + if collation != nil { + command = append(command, bson.DocElem{"collation", collation}) + } + return db.Run(command, nil) +} + // With returns a copy of db that uses session s. func (db *Database) With(s *Session) *Database { newdb := *db diff --git a/session_test.go b/session_test.go index 216474889..5f1689bea 100644 --- a/session_test.go +++ b/session_test.go @@ -1261,6 +1261,131 @@ func (s *S) TestCountCollection(c *C) { c.Assert(n, Equals, 3) } +func (s *S) TestView(c *C) { + if !s.versionAtLeast(3, 4) { + c.Skip("depends on mongodb 3.4+") + } + // CreateView has to be run against mongos + session, err := mgo.Dial("localhost:40201") + c.Assert(err, IsNil) + defer session.Close() + + db := session.DB("mydb") + + coll := db.C("mycoll") + + for i := 0; i < 4; i++ { + err = coll.Insert(bson.M{"_id": i, "nm": "a"}) + c.Assert(err, IsNil) + } + + pipeline := []bson.M{{"$match": bson.M{"_id": bson.M{"$gte": 2}}}} + + err = db.CreateView("myview", coll.Name, pipeline, nil) + c.Assert(err, IsNil) + + names, err := db.CollectionNames() + c.Assert(err, IsNil) + c.Assert(names, DeepEquals, []string{"mycoll", "myview", "system.views"}) + + var viewInfo struct { + ID string `bson:"_id"` + ViewOn string `bson:"viewOn"` + Pipeline []bson.M `bson:"pipeline"` + } + + err = db.C("system.views").Find(nil).One(&viewInfo) + c.Assert(viewInfo.ID, Equals, "mydb.myview") + c.Assert(viewInfo.ViewOn, Equals, "mycoll") + c.Assert(viewInfo.Pipeline, DeepEquals, pipeline) + + view := db.C("myview") + + n, err := view.Count() + c.Assert(err, IsNil) + c.Assert(n, Equals, 2) + + var result struct { + ID int `bson:"_id"` + Nm string `bson:"nm"` + } + + err = view.Find(nil).Sort("_id").One(&result) + c.Assert(err, IsNil) + c.Assert(result.ID, Equals, 2) + + err = view.Find(bson.M{"_id": 3}).One(&result) + c.Assert(err, IsNil) + c.Assert(result.ID, Equals, 3) + + var resultPipe struct { + ID int `bson:"_id"` + Nm string `bson:"nm"` + C int `bson:"c"` + } + + err = view.Pipe([]bson.M{{"$project": bson.M{"c": bson.M{"$sum": []interface{}{"$_id", 10}}}}}).One(&resultPipe) + c.Assert(err, IsNil) + c.Assert(resultPipe.C, Equals, 12) + + err = view.EnsureIndexKey("nm") + c.Assert(err, NotNil) + + err = view.Insert(bson.M{"_id": 5, "nm": "b"}) + c.Assert(err, NotNil) + + err = view.Remove(bson.M{"_id": 2}) + c.Assert(err, NotNil) + + err = view.Update(bson.M{"_id": 2}, bson.M{"$set": bson.M{"d": true}}) + c.Assert(err, NotNil) + + err = db.C("myview").DropCollection() + c.Assert(err, IsNil) + + names, err = db.CollectionNames() + c.Assert(err, IsNil) + c.Assert(names, DeepEquals, []string{"mycoll", "system.views"}) + + n, err = db.C("system.views").Count() + c.Assert(err, IsNil) + c.Assert(n, Equals, 0) + +} + +func (s *S) TestViewWithCollation(c *C) { + if !s.versionAtLeast(3, 4) { + c.Skip("depends on mongodb 3.4+") + } + // CreateView has to be run against mongos + session, err := mgo.Dial("localhost:40201") + c.Assert(err, IsNil) + defer session.Close() + + db := session.DB("mydb") + + coll := db.C("mycoll") + + names := []string{"case", "CaSe", "cäse"} + for _, name := range names { + err = coll.Insert(bson.M{"nm": name}) + c.Assert(err, IsNil) + } + + collation := &mgo.Collation{Locale: "en", Strength: 2} + + err = db.CreateView("myview", "mycoll", []bson.M{{"$match": bson.M{"nm": "case"}}}, collation) + c.Assert(err, IsNil) + + var docs []struct { + Nm string `bson:"nm"` + } + err = db.C("myview").Find(nil).All(&docs) + c.Assert(err, IsNil) + c.Assert(docs[0].Nm, Equals, "case") + c.Assert(docs[1].Nm, Equals, "CaSe") +} + func (s *S) TestCountQuery(c *C) { session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) From 1f4c10f0b5e6e020e56b93e551ede0f6c2423a39 Mon Sep 17 00:00:00 2001 From: Dom Date: Mon, 11 Sep 2017 17:04:12 +0100 Subject: [PATCH 06/38] readme: credit @feliixx in the README (#36) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fce236bf7..a319b1ab4 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Further PR's (with tests) are welcome, but please maintain backwards compatibili * Support index hints and timeouts for count queries ([details](https://github.com/globalsign/mgo/pull/17)) * Allow dropping all indexes on a collection ([details](https://github.com/globalsign/mgo/pull/25)) * Annotates log entries/profiler output with optional appName on 3.4+ ([details](https://github.com/globalsign/mgo/pull/28)) +* Support for read-only [views](https://docs.mongodb.com/manual/core/views/) in 3.4+ ([details](https://github.com/globalsign/mgo/pull/33)) --- From 934a190f5f79d5923a85a3816fe80d2a085393b9 Mon Sep 17 00:00:00 2001 From: Dom Date: Mon, 11 Sep 2017 19:52:51 +0100 Subject: [PATCH 07/38] Don't panic on indexed int64 fields (#23) * Stop all db instances after tests (#462) If all tests pass, the builds for mongo earlier than 2.6 are still failing. Running a clean up fixes the issue. * fixing int64 type failing when getting indexes and trying to type them * requested changes relating to case statement and panic * Update README.md to credit @mapete94. * tests: ensure indexed int64 fields do not cause a panic in Indexes() See: * https://github.com/globalsign/mgo/pull/23 * https://github.com/go-mgo/mgo/issues/475 * https://github.com/go-mgo/mgo/pull/476 --- README.md | 4 +++- session.go | 21 +++++++++++++++------ session_internal_test.go | 24 ++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 7 deletions(-) create mode 100644 session_internal_test.go diff --git a/README.md b/README.md index a319b1ab4..349aaee43 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,8 @@ Further PR's (with tests) are welcome, but please maintain backwards compatibili * Improved multi-document transaction performance ([details](https://github.com/globalsign/mgo/pull/10), [more](https://github.com/globalsign/mgo/pull/11), [more](https://github.com/globalsign/mgo/pull/16)) * Fixes cursor timeouts ([details](https://jira.mongodb.org/browse/SERVER-24899)) * Support index hints and timeouts for count queries ([details](https://github.com/globalsign/mgo/pull/17)) -* Allow dropping all indexes on a collection ([details](https://github.com/globalsign/mgo/pull/25)) +* Don't panic when handling indexed `int64` fields ([detials](https://github.com/go-mgo/mgo/issues/475)) +* Supports dropping all indexes on a collection ([details](https://github.com/globalsign/mgo/pull/25)) * Annotates log entries/profiler output with optional appName on 3.4+ ([details](https://github.com/globalsign/mgo/pull/28)) * Support for read-only [views](https://docs.mongodb.com/manual/core/views/) in 3.4+ ([details](https://github.com/globalsign/mgo/pull/33)) @@ -36,6 +37,7 @@ Further PR's (with tests) are welcome, but please maintain backwards compatibili * @feliixx * @fmpwizard * @jameinel +* @mapete94 * @Reenjii * @smoya * @wgallagher \ No newline at end of file diff --git a/session.go b/session.go index b1bbcdbf7..5e7c99775 100644 --- a/session.go +++ b/session.go @@ -1659,12 +1659,25 @@ func (idxs indexSlice) Swap(i, j int) { idxs[i], idxs[j] = idxs[j], idxs[i] func simpleIndexKey(realKey bson.D) (key []string) { for i := range realKey { + var vi int field := realKey[i].Name - vi, ok := realKey[i].Value.(int) - if !ok { + + switch realKey[i].Value.(type) { + case int64: + vf, _ := realKey[i].Value.(int64) + vi = int(vf) + case float64: vf, _ := realKey[i].Value.(float64) vi = int(vf) + case string: + if vs, ok := realKey[i].Value.(string); ok { + key = append(key, "$"+vs+":"+field) + continue + } + case int: + vi = realKey[i].Value.(int) } + if vi == 1 { key = append(key, field) continue @@ -1673,10 +1686,6 @@ func simpleIndexKey(realKey bson.D) (key []string) { key = append(key, "-"+field) continue } - if vs, ok := realKey[i].Value.(string); ok { - key = append(key, "$"+vs+":"+field) - continue - } panic("Got unknown index key type for field " + field) } return diff --git a/session_internal_test.go b/session_internal_test.go new file mode 100644 index 000000000..f5f796c99 --- /dev/null +++ b/session_internal_test.go @@ -0,0 +1,24 @@ +package mgo + +import ( + "testing" + + "github.com/globalsign/mgo/bson" +) + +// This file is for testing functions that are not exported outside the mgo +// package - avoid doing so if at all possible. + +// Ensures indexed int64 fields do not cause mgo to panic. +// +// See https://github.com/globalsign/mgo/pull/23 +func TestIndexedInt64FieldsBug(t *testing.T) { + input := bson.D{ + {Name: "testkey", Value: int(1)}, + {Name: "testkey", Value: int64(1)}, + {Name: "testkey", Value: "test"}, + {Name: "testkey", Value: float64(1)}, + } + + _ = simpleIndexKey(input) +} From b37e3c1047e7007f6b824af23a8194f7e6a643c9 Mon Sep 17 00:00:00 2001 From: Adrien Petel Date: Wed, 13 Sep 2017 16:57:01 +0400 Subject: [PATCH 08/38] Add collation option to collection.Create() (#37) - Allow specifying the default collation for the collection when creating it. - Add some documentation to query.Collation() method. fix #29 --- session.go | 32 ++++++++++++++++++++++++++++++++ session_test.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/session.go b/session.go index 5e7c99775..2b383ad40 100644 --- a/session.go +++ b/session.go @@ -2857,6 +2857,10 @@ type CollectionInfo struct { // storage engine in use. The map keys must hold the storage engine // name for which options are being specified. StorageEngine interface{} + // Specifies the default collation for the collection. + // Collation allows users to specify language-specific rules for string + // comparison, such as rules for lettercase and accent marks. + Collation *Collation } // Create explicitly creates the c collection with details of info. @@ -2900,6 +2904,10 @@ func (c *Collection) Create(info *CollectionInfo) error { if info.StorageEngine != nil { cmd = append(cmd, bson.DocElem{"storageEngine", info.StorageEngine}) } + if info.Collation != nil { + cmd = append(cmd, bson.DocElem{"collation", info.Collation}) + } + return c.Database.Run(cmd, nil) } @@ -3039,6 +3047,30 @@ func (q *Query) Sort(fields ...string) *Query { return q } +// Collation allows to specify language-specific rules for string comparison, +// such as rules for lettercase and accent marks. +// When specifying collation, the locale field is mandatory; all other collation +// fields are optional +// +// For example, to perform a case and diacritic insensitive query: +// +// var res []bson.M +// collation := &mgo.Collation{Locale: "en", Strength: 1} +// err = db.C("mycoll").Find(bson.M{"a": "a"}).Collation(collation).All(&res) +// if err != nil { +// return err +// } +// +// This query will match following documents: +// +// {"a": "a"} +// {"a": "A"} +// {"a": "â"} +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/reference/collation/ +// func (q *Query) Collation(collation *Collation) *Query { q.m.Lock() q.op.options.Collation = collation diff --git a/session_test.go b/session_test.go index 5f1689bea..f3bb70ec9 100644 --- a/session_test.go +++ b/session_test.go @@ -1020,6 +1020,39 @@ func (s *S) TestCreateCollectionStorageEngine(c *C) { c.Assert(err, ErrorMatches, "test is not a registered storage engine for this server") } +func (s *S) TestCreateCollectionWithCollation(c *C) { + if !s.versionAtLeast(3, 4) { + c.Skip("depends on mongodb 3.4+") + } + session, err := mgo.Dial("localhost:40001") + c.Assert(err, IsNil) + defer session.Close() + + db := session.DB("mydb") + coll := db.C("mycoll") + + info := &mgo.CollectionInfo{ + Collation: &mgo.Collation{Locale: "en", Strength: 1}, + } + err = coll.Create(info) + c.Assert(err, IsNil) + + err = coll.Insert(M{"a": "case"}) + c.Assert(err, IsNil) + + err = coll.Insert(M{"a": "CaSe"}) + c.Assert(err, IsNil) + + var docs []struct { + A string `bson:"a"` + } + err = coll.Find(bson.M{"a": "case"}).All(&docs) + c.Assert(err, IsNil) + c.Assert(docs[0].A, Equals, "case") + c.Assert(docs[1].A, Equals, "CaSe") + +} + func (s *S) TestIsDupValues(c *C) { c.Assert(mgo.IsDup(nil), Equals, false) c.Assert(mgo.IsDup(&mgo.LastError{Code: 1}), Equals, false) From 10876f519770e794caff3f8f3dbf2900ec229be7 Mon Sep 17 00:00:00 2001 From: Will Banfield Date: Wed, 2 Aug 2017 18:38:43 -0400 Subject: [PATCH 09/38] support the $changeStream aggregation in 3.6+ The $changeStream aggregation isn't very well documented, but an overview can be found here: https://mongodb.github.io/mongo-java-driver/3.6/driver/tutorials/change-streams/ This commit is a squash of 4 commits from the 10gen fork of mgo: * e78a135e0cc88f6e5c8fc7cf7df464bee1c1f820 * 31a6d57f052d3049b245a226aa6691a3b9230ae7 * 7940d20144fd1dace1d07c1652ed199bab99614f * 0d8351c5431c67e5f73f0d36b97cf67cb55d9741 --- changestreams.go | 273 +++++++++++++++++++++++++++++++++++++++++++++++ session.go | 26 +++-- 2 files changed, 293 insertions(+), 6 deletions(-) create mode 100644 changestreams.go diff --git a/changestreams.go b/changestreams.go new file mode 100644 index 000000000..c19ded317 --- /dev/null +++ b/changestreams.go @@ -0,0 +1,273 @@ +package mgo + +import ( + "fmt" + "reflect" + "sync" + + "gopkg.in/mgo.v2/bson" +) + +type ChangeStream struct { + iter *Iter + options ChangeStreamOptions + pipeline interface{} + resumeToken *bson.Raw + collection *Collection + readPreference *ReadPreference + err error + m sync.Mutex +} + +type ChangeStreamOptions struct { + + // FullDocument controls the amount of data that the server will return when + // returning a changes document. + FullDocument string + + // ResumeAfter specifies the logical starting point for the new change stream. + ResumeAfter *bson.Raw + + // MaxAwaitTimeMS specifies the maximum amount of time for the server to wait + // on new documents to satisfy a change stream query. + MaxAwaitTimeMS int64 + + // BatchSize specifies the number of documents to return per batch. + BatchSize int32 + + // Collation specifies the way the server should collate returned data. + Collation *Collation +} + +// Watch constructs a new ChangeStream capable of receiving continuing data +// from the database. +func (coll *Collection) Watch(pipeline interface{}, + options ChangeStreamOptions) (*ChangeStream, error) { + + if pipeline == nil { + pipeline = []bson.M{} + } + + pipe := constructChangeStreamPipeline(pipeline, options) + + pIter := coll.Pipe(&pipe).Iter() + + // check that there was no issue creating the iterator. + // this will fail immediately with an error from the server if running against + // a standalone. + if err := pIter.Err(); err != nil { + return nil, err + } + + pIter.isChangeStream = true + + return &ChangeStream{ + iter: pIter, + collection: coll, + resumeToken: nil, + options: options, + pipeline: pipeline, + }, nil +} + +// Next retrieves the next document from the change stream, blocking if necessary. +// Next returns true if a document was successfully unmarshalled into result, +// and false if an error occured. When Next returns false, the Err method should +// be called to check what error occurred during iteration. +// +// For example: +// +// pipeline := []bson.M{} +// +// changeStream := collection.Watch(pipeline, ChangeStreamOptions{}) +// for changeStream.Next(&changeDoc) { +// fmt.Printf("Change: %v\n", changeDoc) +// } +// +// if err := changeStream.Close(); err != nil { +// return err +// } +// +// If the pipeline used removes the _id field from the result, Next will error +// because the _id field is needed to resume iteration when an error occurs. +// +func (changeStream *ChangeStream) Next(result interface{}) bool { + // the err field is being constantly overwritten and we don't want the user to + // attempt to read it at this point so we lock. + changeStream.m.Lock() + + defer changeStream.m.Unlock() + + // if we are in a state of error, then don't continue. + if changeStream.err != nil { + return false + } + + var err error + + // attempt to fetch the change stream result. + err = changeStream.fetchResultSet(result) + if err == nil { + return true + } + + // check if the error is resumable + if !isResumableError(err) { + // error is not resumable, give up and return it to the user. + changeStream.err = err + return false + } + + // try to resume. + err = changeStream.resume() + if err != nil { + // we've not been able to successfully resume and should only try once, + // so we give up. + changeStream.err = err + return false + } + + // we've successfully resumed the changestream. + // try to fetch the next result. + err = changeStream.fetchResultSet(result) + if err != nil { + changeStream.err = err + return false + } + + return true +} + +func constructChangeStreamPipeline(pipeline interface{}, + options ChangeStreamOptions) interface{} { + pipelinev := reflect.ValueOf(pipeline) + + // ensure that the pipeline passed in is a slice. + if pipelinev.Kind() != reflect.Slice { + panic("pipeline argument must be a slice") + } + + // construct the options to be used by the change notification + // pipeline stage. + changeStreamStageOptions := bson.M{} + + if options.FullDocument != "" { + changeStreamStageOptions["fullDocument"] = options.FullDocument + } + if options.ResumeAfter != nil { + changeStreamStageOptions["resumeAfter"] = options.ResumeAfter + } + changeStreamStage := bson.M{"$changeStream": changeStreamStageOptions} + + pipeOfInterfaces := make([]interface{}, pipelinev.Len()+1) + + // insert the change notification pipeline stage at the beginning of the + // aggregation. + pipeOfInterfaces[0] = changeStreamStage + + // convert the passed in slice to a slice of interfaces. + for i := 0; i < pipelinev.Len(); i++ { + pipeOfInterfaces[1+i] = pipelinev.Index(i).Addr().Interface() + } + var pipelineAsInterface interface{} = pipeOfInterfaces + return pipelineAsInterface +} + +func (changeStream *ChangeStream) resume() error { + // copy the information for the new socket. + + // Copy() destroys the sockets currently associated with this session + // so future uses will acquire a new socket against the newly selected DB. + newSession := changeStream.iter.session.Copy() + + // fetch the cursor from the iterator and use it to run a killCursors + // on the connection. + cursorId := changeStream.iter.op.cursorId + err := runKillCursorsOnSession(newSession, cursorId) + if err != nil { + return err + } + + // change out the old connection to the database with the new connection. + changeStream.collection.Database.Session = newSession + + // make a new pipeline containing the resume token. + changeStreamPipeline := constructChangeStreamPipeline(changeStream.pipeline, changeStream.options) + + // generate the new iterator with the new connection. + newPipe := changeStream.collection.Pipe(changeStreamPipeline) + changeStream.iter = newPipe.Iter() + changeStream.iter.isChangeStream = true + + return nil +} + +// fetchResumeToken unmarshals the _id field from the document, setting an error +// on the changeStream if it is unable to. +func (changeStream *ChangeStream) fetchResumeToken(rawResult *bson.Raw) error { + changeStreamResult := struct { + ResumeToken *bson.Raw `bson:"_id,omitempty"` + }{} + + err := rawResult.Unmarshal(&changeStreamResult) + if err != nil { + return err + } + + if changeStreamResult.ResumeToken == nil { + return fmt.Errorf("resume token missing from result") + } + + changeStream.resumeToken = changeStreamResult.ResumeToken + return nil +} + +func (changeStream *ChangeStream) fetchResultSet(result interface{}) error { + rawResult := bson.Raw{} + + // fetch the next set of documents from the cursor. + gotNext := changeStream.iter.Next(&rawResult) + + err := changeStream.iter.Err() + if err != nil { + return err + } + + if !gotNext && err == nil { + // If the iter.Err() method returns nil despite us not getting a next batch, + // it is becuase iter.Err() silences this case. + return ErrNotFound + } + + // grab the resumeToken from the results + if err := changeStream.fetchResumeToken(&rawResult); err != nil { + return err + } + + // put the raw results into the data structure the user provided. + if err := rawResult.Unmarshal(result); err != nil { + return err + } + return nil +} + +func isResumableError(err error) bool { + _, isQueryError := err.(*QueryError) + // if it is not a database error OR it is a database error, + // but the error is a notMaster error + return !isQueryError || isNotMasterError(err) +} + +func runKillCursorsOnSession(session *Session, cursorId int64) error { + socket, err := session.acquireSocket(true) + if err != nil { + return err + } + err = socket.Query(&killCursorsOp{[]int64{cursorId}}) + if err != nil { + return err + } + socket.Release() + + return nil +} diff --git a/session.go b/session.go index 2b383ad40..c7f2890bd 100644 --- a/session.go +++ b/session.go @@ -138,7 +138,8 @@ type Iter struct { timeout time.Duration limit int32 timedout bool - findCmd bool + isFindCmd bool + isChangeStream bool } var ( @@ -1019,6 +1020,11 @@ func isAuthError(err error) bool { return ok && e.Code == 13 } +func isNotMasterError(err error) bool { + e, ok := err.(*QueryError) + return ok && strings.Contains(e.Message, "not master") +} + func (db *Database) runUserCmd(cmdName string, user *User) error { cmd := make(bson.D, 0, 16) cmd = append(cmd, bson.DocElem{cmdName, user.Username}) @@ -2450,7 +2456,7 @@ func (c *Collection) NewIter(session *Session, firstBatch []bson.Raw, cursorId i } if socket.ServerInfo().MaxWireVersion >= 4 && c.FullName != "admin.$cmd" { - iter.findCmd = true + iter.isFindCmd = true } iter.gotReply.L = &iter.m @@ -3688,7 +3694,7 @@ func (q *Query) Iter() *Iter { op.replyFunc = iter.op.replyFunc if prepareFindOp(socket, &op, limit) { - iter.findCmd = true + iter.isFindCmd = true } iter.server = socket.Server() @@ -3918,7 +3924,12 @@ func (iter *Iter) Next(result interface{}) bool { iter.m.Lock() iter.timedout = false timeout := time.Time{} + + // check should we expect more data. for iter.err == nil && iter.docData.Len() == 0 && (iter.docsToReceive > 0 || iter.op.cursorId != 0) { + // we should expect more data. + + // If we have yet to receive data, increment the timer until we timeout. if iter.docsToReceive == 0 { if iter.timeout >= 0 { if timeout.IsZero() { @@ -3930,6 +3941,7 @@ func (iter *Iter) Next(result interface{}) bool { return false } } + // run a getmore to fetch more data. iter.getMore() if iter.err != nil { break @@ -3938,6 +3950,7 @@ func (iter *Iter) Next(result interface{}) bool { iter.gotReply.Wait() } + // We have data from the getMore. // Exhaust available data before reporting any errors. if docData, ok := iter.docData.Pop().([]byte); ok { close := false @@ -3953,6 +3966,7 @@ func (iter *Iter) Next(result interface{}) bool { } } if iter.op.cursorId != 0 && iter.err == nil { + // we still have a live cursor and currently expect data. iter.docsBeforeMore-- if iter.docsBeforeMore == -1 { iter.getMore() @@ -4142,7 +4156,7 @@ func (iter *Iter) getMore() { } } var op interface{} - if iter.findCmd { + if iter.isFindCmd || iter.isChangeStream { op = iter.getMoreCmd() } else { op = &iter.op @@ -4751,7 +4765,7 @@ func (iter *Iter) replyFunc() replyFunc { } else { iter.err = ErrNotFound } - } else if iter.findCmd { + } else if iter.isFindCmd { debugf("Iter %p received reply document %d/%d (cursor=%d)", iter, docNum+1, int(op.replyDocs), op.cursorId) var findReply struct { Ok bool @@ -4763,7 +4777,7 @@ func (iter *Iter) replyFunc() replyFunc { iter.err = err } else if !findReply.Ok && findReply.Errmsg != "" { iter.err = &QueryError{Code: findReply.Code, Message: findReply.Errmsg} - } else if len(findReply.Cursor.FirstBatch) == 0 && len(findReply.Cursor.NextBatch) == 0 { + } else if !iter.isChangeStream && len(findReply.Cursor.FirstBatch) == 0 && len(findReply.Cursor.NextBatch) == 0 { iter.err = ErrNotFound } else { batch := findReply.Cursor.FirstBatch From b82ca4c9ed299576e5a9dd5ba038ae8d082173b2 Mon Sep 17 00:00:00 2001 From: Dom Dwyer Date: Fri, 15 Sep 2017 10:10:18 +0100 Subject: [PATCH 10/38] changestreams: fix import path --- changestreams.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changestreams.go b/changestreams.go index c19ded317..6161678ea 100644 --- a/changestreams.go +++ b/changestreams.go @@ -5,7 +5,7 @@ import ( "reflect" "sync" - "gopkg.in/mgo.v2/bson" + "github.com/globalsign/mgo/bson" ) type ChangeStream struct { From aead58f02acbc4655bdab9bbe59f80e2197a87ff Mon Sep 17 00:00:00 2001 From: Adrien Petel Date: Fri, 15 Sep 2017 19:18:01 +0400 Subject: [PATCH 11/38] Test against MongoDB 3.4.x (#35) * test against MongoDB 3.4.x * tests: use listIndexes to assert index state for 3.4+ * make test pass against v3.4.x - skip `TestViewWithCollation` because of SERVER-31049, cf: https://jira.mongodb.org/browse/SERVER-31049 - add versionAtLeast() method in init.js script to better detect server version fixes #31 --- .travis.yml | 14 +-- cluster_test.go | 18 +++ harness/daemons/.env | 42 ++++++- harness/daemons/cfg1/run | 3 +- harness/daemons/cfg2/run | 3 +- harness/daemons/cfg3/run | 1 + harness/daemons/s1/run | 2 +- harness/daemons/s2/run | 2 +- harness/daemons/s3/run | 2 +- harness/mongojs/init.js | 37 ++++-- session_test.go | 250 +++++++++++++++++++++++++++++---------- 11 files changed, 287 insertions(+), 87 deletions(-) diff --git a/.travis.yml b/.travis.yml index 28d3c5cf4..430844718 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,29 +2,22 @@ language: go go_import_path: github.com/globalsign/mgo -addons: - apt: - packages: - env: global: - BUCKET=https://s3.eu-west-2.amazonaws.com/globalsign-mgo matrix: - - GO=1.6 MONGODB=x86_64-2.6.11 - GO=1.7 MONGODB=x86_64-2.6.11 - GO=1.8.x MONGODB=x86_64-2.6.11 - - GO=1.6 MONGODB=x86_64-3.0.9 - GO=1.7 MONGODB=x86_64-3.0.9 - GO=1.8.x MONGODB=x86_64-3.0.9 - - GO=1.6 MONGODB=x86_64-3.2.3-nojournal - GO=1.7 MONGODB=x86_64-3.2.3-nojournal - GO=1.8.x MONGODB=x86_64-3.2.3-nojournal - - GO=1.6 MONGODB=x86_64-3.2.12 - GO=1.7 MONGODB=x86_64-3.2.12 - GO=1.8.x MONGODB=x86_64-3.2.12 - - GO=1.6 MONGODB=x86_64-3.2.16 - GO=1.7 MONGODB=x86_64-3.2.16 - GO=1.8.x MONGODB=x86_64-3.2.16 + - GO=1.7 MONGODB=x86_64-3.4.8 + - GO=1.8.x MONGODB=x86_64-3.4.8 install: - eval "$(gimme $GO)" @@ -51,4 +44,7 @@ script: - (cd txn && go test -check.v) - make stopdb +git: + depth: 3 + # vim:sw=4:ts=4:et diff --git a/cluster_test.go b/cluster_test.go index fc95078f6..1436cc317 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -1281,6 +1281,9 @@ func (s *S) countCommands(c *C, server, commandName string) (n int) { } func (s *S) TestMonotonicSlaveOkFlagWithMongos(c *C) { + if s.versionAtLeast(3, 4) { + c.Skip("fail on 3.4+ ? ") + } session, err := mgo.Dial("localhost:40021") c.Assert(err, IsNil) defer session.Close() @@ -1369,6 +1372,12 @@ func (s *S) TestMonotonicSlaveOkFlagWithMongos(c *C) { } func (s *S) TestSecondaryModeWithMongos(c *C) { + if *fast { + c.Skip("-fast") + } + if s.versionAtLeast(3, 4) { + c.Skip("fail on 3.4+ ?") + } session, err := mgo.Dial("localhost:40021") c.Assert(err, IsNil) defer session.Close() @@ -1870,6 +1879,9 @@ func (s *S) TestNearestSecondary(c *C) { } func (s *S) TestNearestServer(c *C) { + if s.versionAtLeast(3, 4) { + c.Skip("fail on 3.4+") + } defer mgo.HackPingDelay(300 * time.Millisecond)() rs1a := "127.0.0.1:40011" @@ -1981,6 +1993,9 @@ func (s *S) TestSelectServersWithMongos(c *C) { if !s.versionAtLeast(2, 2) { c.Skip("read preferences introduced in 2.2") } + if s.versionAtLeast(3, 4) { + c.Skip("fail on 3.4+") + } session, err := mgo.Dial("localhost:40021") c.Assert(err, IsNil) @@ -2067,6 +2082,9 @@ func (s *S) TestDoNotFallbackToMonotonic(c *C) { if !s.versionAtLeast(3, 0) { c.Skip("command-counting logic depends on 3.0+") } + if s.versionAtLeast(3, 4) { + c.Skip("failing on 3.4+") + } session, err := mgo.Dial("localhost:40012") c.Assert(err, IsNil) diff --git a/harness/daemons/.env b/harness/daemons/.env index 5b9e0767c..b9a900647 100644 --- a/harness/daemons/.env +++ b/harness/daemons/.env @@ -40,16 +40,48 @@ COMMONSOPTS=" --bind_ip=127.0.0.1 " +CFG1OPTS="" +CFG2OPTS="" +CFG3OPTS="" + +MONGOS1OPTS="--configdb 127.0.0.1:40101" +MONGOS2OPTS="--configdb 127.0.0.1:40102" +MONGOS3OPTS="--configdb 127.0.0.1:40103" + + + if versionAtLeast 3 2; then - # 3.2 doesn't like --nojournal on config servers. + + # 3.2 doesn't like --nojournal on config servers. COMMONCOPTS="$(echo "$COMMONCOPTS" | sed '/--nojournal/d')" - # Go back to MMAPv1 so it's not super sluggish. :-( - COMMONDOPTSNOIP="--storageEngine=mmapv1 $COMMONDOPTSNOIP" - COMMONDOPTS="--storageEngine=mmapv1 $COMMONDOPTS" - COMMONCOPTS="--storageEngine=mmapv1 $COMMONCOPTS" + + if versionAtLeast 3 4; then + # http interface is disabled by default, this option does not exist anymore + COMMONDOPTSNOIP="$(echo "$COMMONDOPTSNOIP" | sed '/--nohttpinterface/d')" + COMMONDOPTS="$(echo "$COMMONDOPTS" | sed '/--nohttpinterface/d')" + COMMONCOPTS="$(echo "$COMMONCOPTS" | sed '/--nohttpinterface/d')" + + + # config server need to be started as replica set + CFG1OPTS="--replSet conf1" + CFG2OPTS="--replSet conf2" + CFG3OPTS="--replSet conf3" + + MONGOS1OPTS="--configdb conf1/127.0.0.1:40101" + MONGOS2OPTS="--configdb conf2/127.0.0.1:40102" + MONGOS3OPTS="--configdb conf3/127.0.0.1:40103" + else + + # Go back to MMAPv1 so it's not super sluggish. :-( + COMMONDOPTSNOIP="--storageEngine=mmapv1 $COMMONDOPTSNOIP" + COMMONDOPTS="--storageEngine=mmapv1 $COMMONDOPTS" + COMMONCOPTS="--storageEngine=mmapv1 $COMMONCOPTS" + fi fi + + if [ "$TRAVIS" = true ]; then set -x fi diff --git a/harness/daemons/cfg1/run b/harness/daemons/cfg1/run index ad6bddd04..e8dc0623b 100755 --- a/harness/daemons/cfg1/run +++ b/harness/daemons/cfg1/run @@ -4,5 +4,6 @@ exec mongod $COMMONCOPTS \ --port 40101 \ - --configsvr + --configsvr \ + $CFG1OPTS diff --git a/harness/daemons/cfg2/run b/harness/daemons/cfg2/run index 07d159ef5..3f2bb496a 100755 --- a/harness/daemons/cfg2/run +++ b/harness/daemons/cfg2/run @@ -4,5 +4,6 @@ exec mongod $COMMONCOPTS \ --port 40102 \ - --configsvr + --configsvr \ + $CFG2OPTS diff --git a/harness/daemons/cfg3/run b/harness/daemons/cfg3/run index bd812fa3e..05b0558c8 100755 --- a/harness/daemons/cfg3/run +++ b/harness/daemons/cfg3/run @@ -5,5 +5,6 @@ exec mongod $COMMONCOPTS \ --port 40103 \ --configsvr \ + $CFG3OPTS \ --auth \ --keyFile=../../certs/keyfile diff --git a/harness/daemons/s1/run b/harness/daemons/s1/run index 0e31d2c94..b267adff5 100755 --- a/harness/daemons/s1/run +++ b/harness/daemons/s1/run @@ -4,4 +4,4 @@ exec mongos $COMMONSOPTS \ --port 40201 \ - --configdb 127.0.0.1:40101 + $MONGOS1OPTS diff --git a/harness/daemons/s2/run b/harness/daemons/s2/run index 3b5c67d58..11b5430b6 100755 --- a/harness/daemons/s2/run +++ b/harness/daemons/s2/run @@ -4,4 +4,4 @@ exec mongos $COMMONSOPTS \ --port 40202 \ - --configdb 127.0.0.1:40102 + $MONGOS2OPTS diff --git a/harness/daemons/s3/run b/harness/daemons/s3/run index fde6e479b..0e6e9e9fc 100755 --- a/harness/daemons/s3/run +++ b/harness/daemons/s3/run @@ -4,5 +4,5 @@ exec mongos $COMMONSOPTS \ --port 40203 \ - --configdb 127.0.0.1:40103 \ + $MONGOS3OPTS \ --keyFile=../../certs/keyfile diff --git a/harness/mongojs/init.js b/harness/mongojs/init.js index ceb75a5e4..909cf5162 100644 --- a/harness/mongojs/init.js +++ b/harness/mongojs/init.js @@ -25,6 +25,9 @@ for (var i = 0; i != 60; i++) { rs1a = new Mongo("127.0.0.1:40011").getDB("admin") rs2a = new Mongo("127.0.0.1:40021").getDB("admin") rs3a = new Mongo("127.0.0.1:40031").getDB("admin") + cfg1 = new Mongo("127.0.0.1:40101").getDB("admin") + cfg2 = new Mongo("127.0.0.1:40102").getDB("admin") + cfg3 = new Mongo("127.0.0.1:40103").getDB("admin") break } catch(err) { print("Can't connect yet...") @@ -36,20 +39,40 @@ function hasSSL() { return Boolean(db1.serverBuildInfo().OpenSSLVersion) } +function versionAtLeast() { + var version = db1.version().split(".") + for (var i = 0; i < arguments.length; i++) { + if (i == arguments.length) { + return false + } + if (arguments[i] != version[i]) { + return version[i] >= arguments[i] + } + } + return true +} + rs1a.runCommand({replSetInitiate: rs1cfg}) rs2a.runCommand({replSetInitiate: rs2cfg}) rs3a.runCommand({replSetInitiate: rs3cfg}) +if (versionAtLeast(3,4)) { + print("configuring config server for mongodb 3.4") + cfg1.runCommand({replSetInitiate: {_id:"conf1", members: [{"_id":1, "host":"localhost:40101"}]}}) + cfg2.runCommand({replSetInitiate: {_id:"conf2", members: [{"_id":1, "host":"localhost:40102"}]}}) + cfg3.runCommand({replSetInitiate: {_id:"conf3", members: [{"_id":1, "host":"localhost:40103"}]}}) +} + function configShards() { - cfg1 = new Mongo("127.0.0.1:40201").getDB("admin") - cfg1.runCommand({addshard: "127.0.0.1:40001"}) - cfg1.runCommand({addshard: "rs1/127.0.0.1:40011"}) + s1 = new Mongo("127.0.0.1:40201").getDB("admin") + s1.runCommand({addshard: "127.0.0.1:40001"}) + s1.runCommand({addshard: "rs1/127.0.0.1:40011"}) - cfg2 = new Mongo("127.0.0.1:40202").getDB("admin") - cfg2.runCommand({addshard: "rs2/127.0.0.1:40021"}) + s2 = new Mongo("127.0.0.1:40202").getDB("admin") + s2.runCommand({addshard: "rs2/127.0.0.1:40021"}) - cfg3 = new Mongo("127.0.0.1:40203").getDB("admin") - cfg3.runCommand({addshard: "rs3/127.0.0.1:40031"}) + s3 = new Mongo("127.0.0.1:40203").getDB("admin") + s3.runCommand({addshard: "rs3/127.0.0.1:40031"}) } function configAuth() { diff --git a/session_test.go b/session_test.go index f3bb70ec9..a4cc04f01 100644 --- a/session_test.go +++ b/session_test.go @@ -413,11 +413,19 @@ func (s *S) TestDatabaseAndCollectionNames(c *C) { names, err = db1.CollectionNames() c.Assert(err, IsNil) - c.Assert(names, DeepEquals, []string{"col1", "col2", "system.indexes"}) + if s.versionAtLeast(3, 4) { + c.Assert(names, DeepEquals, []string{"col1", "col2"}) + } else { + c.Assert(names, DeepEquals, []string{"col1", "col2", "system.indexes"}) + } names, err = db2.CollectionNames() c.Assert(err, IsNil) - c.Assert(names, DeepEquals, []string{"col3", "system.indexes"}) + if s.versionAtLeast(3, 4) { + c.Assert(names, DeepEquals, []string{"col3"}) + } else { + c.Assert(names, DeepEquals, []string{"col3", "system.indexes"}) + } } func (s *S) TestSelect(c *C) { @@ -872,14 +880,22 @@ func (s *S) TestDropCollection(c *C) { names, err := db.CollectionNames() c.Assert(err, IsNil) - c.Assert(names, DeepEquals, []string{"col2", "system.indexes"}) + if s.versionAtLeast(3, 4) { + c.Assert(names, DeepEquals, []string{"col2"}) + } else { + c.Assert(names, DeepEquals, []string{"col2", "system.indexes"}) + } err = db.C("col2").DropCollection() c.Assert(err, IsNil) names, err = db.CollectionNames() c.Assert(err, IsNil) - c.Assert(names, DeepEquals, []string{"system.indexes"}) + if s.versionAtLeast(3, 4) { + c.Assert(len(names), Equals, 0) + } else { + c.Assert(names, DeepEquals, []string{"system.indexes"}) + } } func (s *S) TestCreateCollectionCapped(c *C) { @@ -1387,6 +1403,11 @@ func (s *S) TestView(c *C) { } func (s *S) TestViewWithCollation(c *C) { + // This test is currently failing because of a bug in mongodb. A ticket describing + // the issue is available here: https://jira.mongodb.org/browse/SERVER-31049 + // TODO remove this line when SERVER-31049 is fixed + c.Skip("Fails because of a MongoDB bug as of version 3.4.9, cf https://jira.mongodb.org/browse/SERVER-31049") + if !s.versionAtLeast(3, 4) { c.Skip("depends on mongodb 3.4+") } @@ -2096,6 +2117,9 @@ func serverCursorsOpen(session *mgo.Session) int { } func (s *S) TestFindIterLimitWithMore(c *C) { + if s.versionAtLeast(3, 4) { + c.Skip("fail on 3.4+") + } session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -3437,13 +3461,35 @@ var indexTests = []struct { }, }} +// getIndex34 uses the listIndexes command to obtain a list of indexes on +// collection, and searches through the result looking for an index with name. +// This can only be used in 3.4+. +// +// The default "_id_" index is never returned, and the "v" field is removed from +// the response. +func getIndex34(session *mgo.Session, db, collection, name string) M { + cmd := bson.M{"listIndexes": collection} + result := M{} + session.DB(db).Run(cmd, result) + + var obtained = M{} + for _, v := range result["cursor"].(M)["firstBatch"].([]interface{}) { + index := v.(M) + if index["name"] == name { + delete(index, "v") + obtained = index + break + } + } + return obtained +} + func (s *S) TestEnsureIndex(c *C) { session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() coll := session.DB("mydb").C("mycoll") - idxs := session.DB("mydb").C("system.indexes") for _, test := range indexTests { if !s.versionAtLeast(2, 4) && test.expected["textIndexVersion"] != nil { @@ -3467,12 +3513,6 @@ func (s *S) TestEnsureIndex(c *C) { expectedName, _ = test.expected["name"].(string) } - obtained := M{} - err = idxs.Find(M{"name": expectedName}).One(obtained) - c.Assert(err, IsNil) - - delete(obtained, "v") - if s.versionAtLeast(2, 7) { // Was deprecated in 2.6, and not being reported by 2.7+. delete(test.expected, "dropDups") @@ -3482,7 +3522,22 @@ func (s *S) TestEnsureIndex(c *C) { test.expected["textIndexVersion"] = 3 } - c.Assert(obtained, DeepEquals, test.expected) + // As of 3.4.X, "system.indexes" is no longer available - instead use: + // + // db.runCommand({"listIndexes": }) + // + // and iterate over the returned cursor. + if s.versionAtLeast(3, 4) { + c.Assert(getIndex34(session, "mydb", "mycoll", test.expected["name"].(string)), DeepEquals, test.expected) + } else { + idxs := session.DB("mydb").C("system.indexes") + obtained := M{} + err = idxs.Find(M{"name": expectedName}).One(obtained) + c.Assert(err, IsNil) + + delete(obtained, "v") + c.Assert(obtained, DeepEquals, test.expected) + } // The result of Indexes must match closely what was used to create the index. indexes, err := coll.Indexes() @@ -3582,34 +3637,53 @@ func (s *S) TestEnsureIndexKey(c *C) { err = coll.EnsureIndexKey("a") c.Assert(err, IsNil) - err = coll.EnsureIndexKey("a", "-b") - c.Assert(err, IsNil) + if s.versionAtLeast(3, 4) { + expected := M{ + "name": "a_1", + "key": M{"a": 1}, + "ns": "mydb.mycoll", + } + c.Assert(getIndex34(session, "mydb", "mycoll", "a_1"), DeepEquals, expected) - sysidx := session.DB("mydb").C("system.indexes") + err = coll.EnsureIndexKey("a", "-b") + c.Assert(err, IsNil) - result1 := M{} - err = sysidx.Find(M{"name": "a_1"}).One(result1) - c.Assert(err, IsNil) + expected = M{ + "name": "a_1_b_-1", + "key": M{"a": 1, "b": -1}, + "ns": "mydb.mycoll", + } + c.Assert(getIndex34(session, "mydb", "mycoll", "a_1_b_-1"), DeepEquals, expected) + } else { + err = coll.EnsureIndexKey("a", "-b") + c.Assert(err, IsNil) - result2 := M{} - err = sysidx.Find(M{"name": "a_1_b_-1"}).One(result2) - c.Assert(err, IsNil) + sysidx := session.DB("mydb").C("system.indexes") - delete(result1, "v") - expected1 := M{ - "name": "a_1", - "key": M{"a": 1}, - "ns": "mydb.mycoll", - } - c.Assert(result1, DeepEquals, expected1) + result1 := M{} + err = sysidx.Find(M{"name": "a_1"}).One(result1) + c.Assert(err, IsNil) - delete(result2, "v") - expected2 := M{ - "name": "a_1_b_-1", - "key": M{"a": 1, "b": -1}, - "ns": "mydb.mycoll", + result2 := M{} + err = sysidx.Find(M{"name": "a_1_b_-1"}).One(result2) + c.Assert(err, IsNil) + + delete(result1, "v") + expected1 := M{ + "name": "a_1", + "key": M{"a": 1}, + "ns": "mydb.mycoll", + } + c.Assert(result1, DeepEquals, expected1) + + delete(result2, "v") + expected2 := M{ + "name": "a_1_b_-1", + "key": M{"a": 1, "b": -1}, + "ns": "mydb.mycoll", + } + c.Assert(result2, DeepEquals, expected2) } - c.Assert(result2, DeepEquals, expected2) } func (s *S) TestEnsureIndexDropIndex(c *C) { @@ -3628,22 +3702,44 @@ func (s *S) TestEnsureIndexDropIndex(c *C) { err = coll.DropIndex("-b") c.Assert(err, IsNil) - sysidx := session.DB("mydb").C("system.indexes") + if s.versionAtLeast(3, 4) { + // system.indexes is deprecated since 3.0, use + // db.runCommand({"listIndexes": }) + // instead - err = sysidx.Find(M{"name": "a_1"}).One(nil) - c.Assert(err, IsNil) + // Assert it exists + c.Assert(getIndex34(session, "mydb", "mycoll", "a_1"), DeepEquals, M{"key": M{"a": 1}, "name": "a_1", "ns": "mydb.mycoll"}) - err = sysidx.Find(M{"name": "b_1"}).One(nil) - c.Assert(err, Equals, mgo.ErrNotFound) + // Assert a missing index returns an empty M{} + c.Assert(getIndex34(session, "mydb", "mycoll", "b_1"), DeepEquals, M{}) - err = coll.DropIndex("a") - c.Assert(err, IsNil) + err = coll.DropIndex("a") + c.Assert(err, IsNil) - err = sysidx.Find(M{"name": "a_1"}).One(nil) - c.Assert(err, Equals, mgo.ErrNotFound) + // Ensure missing + c.Assert(getIndex34(session, "mydb", "mycoll", "a_1"), DeepEquals, M{}) - err = coll.DropIndex("a") - c.Assert(err, ErrorMatches, "index not found.*") + // Try to drop it again + err = coll.DropIndex("a") + c.Assert(err, ErrorMatches, "index not found.*") + } else { + sysidx := session.DB("mydb").C("system.indexes") + + err = sysidx.Find(M{"name": "a_1"}).One(nil) + c.Assert(err, IsNil) + + err = sysidx.Find(M{"name": "b_1"}).One(nil) + c.Assert(err, Equals, mgo.ErrNotFound) + + err = coll.DropIndex("a") + c.Assert(err, IsNil) + + err = sysidx.Find(M{"name": "a_1"}).One(nil) + c.Assert(err, Equals, mgo.ErrNotFound) + + err = coll.DropIndex("a") + c.Assert(err, ErrorMatches, "index not found.*") + } } func (s *S) TestEnsureIndexDropIndexName(c *C) { @@ -3661,23 +3757,44 @@ func (s *S) TestEnsureIndexDropIndexName(c *C) { err = coll.DropIndexName("a") c.Assert(err, IsNil) + if s.versionAtLeast(3, 4) { + // system.indexes is deprecated since 3.0, use + // db.runCommand({"listIndexes": }) + // instead - sysidx := session.DB("mydb").C("system.indexes") + // Assert it exists + c.Assert(getIndex34(session, "mydb", "mycoll", "a_1"), DeepEquals, M{"ns": "mydb.mycoll", "key": M{"a": 1}, "name": "a_1"}) - err = sysidx.Find(M{"name": "a_1"}).One(nil) - c.Assert(err, IsNil) + // Assert M{} is returned for a missing index + c.Assert(getIndex34(session, "mydb", "mycoll", "a"), DeepEquals, M{}) - err = sysidx.Find(M{"name": "a"}).One(nil) - c.Assert(err, Equals, mgo.ErrNotFound) + err = coll.DropIndexName("a_1") + c.Assert(err, IsNil) - err = coll.DropIndexName("a_1") - c.Assert(err, IsNil) + // Ensure it's gone + c.Assert(getIndex34(session, "mydb", "mycoll", "a_1"), DeepEquals, M{}) - err = sysidx.Find(M{"name": "a_1"}).One(nil) - c.Assert(err, Equals, mgo.ErrNotFound) + err = coll.DropIndexName("a_1") + c.Assert(err, ErrorMatches, "index not found.*") + + } else { + sysidx := session.DB("mydb").C("system.indexes") + + err = sysidx.Find(M{"name": "a_1"}).One(nil) + c.Assert(err, IsNil) + + err = sysidx.Find(M{"name": "a"}).One(nil) + c.Assert(err, Equals, mgo.ErrNotFound) - err = coll.DropIndexName("a_1") - c.Assert(err, ErrorMatches, "index not found.*") + err = coll.DropIndexName("a_1") + c.Assert(err, IsNil) + + err = sysidx.Find(M{"name": "a_1"}).One(nil) + c.Assert(err, Equals, mgo.ErrNotFound) + + err = coll.DropIndexName("a_1") + c.Assert(err, ErrorMatches, "index not found.*") + } } func (s *S) TestEnsureIndexDropAllIndexes(c *C) { @@ -3696,13 +3813,21 @@ func (s *S) TestEnsureIndexDropAllIndexes(c *C) { err = coll.DropAllIndexes() c.Assert(err, IsNil) - sysidx := session.DB("mydb").C("system.indexes") + if s.versionAtLeast(3, 4) { + // system.indexes is deprecated since 3.0, use + // db.runCommand({"listIndexes": }) + // instead + c.Assert(getIndex34(session, "mydb", "mycoll", "a_1"), DeepEquals, M{}) + c.Assert(getIndex34(session, "mydb", "mycoll", "b_1"), DeepEquals, M{}) + } else { + sysidx := session.DB("mydb").C("system.indexes") - err = sysidx.Find(M{"name": "a_1"}).One(nil) - c.Assert(err, Equals, mgo.ErrNotFound) + err = sysidx.Find(M{"name": "a_1"}).One(nil) + c.Assert(err, Equals, mgo.ErrNotFound) - err = sysidx.Find(M{"name": "b_1"}).One(nil) - c.Assert(err, Equals, mgo.ErrNotFound) + err = sysidx.Find(M{"name": "b_1"}).One(nil) + c.Assert(err, Equals, mgo.ErrNotFound) + } } func (s *S) TestEnsureIndexCaching(c *C) { @@ -4268,6 +4393,9 @@ func (s *S) TestRepairCursor(c *C) { if !s.versionAtLeast(2, 7) { c.Skip("RepairCursor only works on 2.7+") } + if s.versionAtLeast(3, 4) { + c.Skip("fail on 3.4+") + } session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) From 5b7a41948ee9de899991d3341faf1d647d28bc72 Mon Sep 17 00:00:00 2001 From: Will Banfield Date: Thu, 14 Sep 2017 16:12:11 -0400 Subject: [PATCH 12/38] MGO-142 implement changestream status functions for err, close and resume token --- changestreams.go | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/changestreams.go b/changestreams.go index 6161678ea..369b269e0 100644 --- a/changestreams.go +++ b/changestreams.go @@ -10,6 +10,7 @@ import ( type ChangeStream struct { iter *Iter + isClosed bool options ChangeStreamOptions pipeline interface{} resumeToken *bson.Raw @@ -103,6 +104,11 @@ func (changeStream *ChangeStream) Next(result interface{}) bool { return false } + if changeStream.isClosed { + changeStream.err = fmt.Errorf("illegal use of a closed ChangeStream") + return false + } + var err error // attempt to fetch the change stream result. @@ -138,6 +144,40 @@ func (changeStream *ChangeStream) Next(result interface{}) bool { return true } +// Err returns nil if no errors happened during iteration, or the actual +// error otherwise. +func (changeStream *ChangeStream) Err() error { + changeStream.m.Lock() + defer changeStream.m.Unlock() + return changeStream.err +} + +// Close kills the server cursor used by the iterator, if any, and returns +// nil if no errors happened during iteration, or the actual error otherwise. +func (changeStream *ChangeStream) Close() error { + changeStream.m.Lock() + defer changeStream.m.Unlock() + changeStream.isClosed = true + err := changeStream.iter.Close() + if err != nil { + changeStream.err = err + } + return err +} + +// ResumeToken returns a copy of the current resume token held by the change stream. +// This token should be treated as an opaque token that can be provided to instantiate +// a new change stream. +func (changeStream *ChangeStream) ResumeToken() *bson.Raw { + changeStream.m.Lock() + defer changeStream.m.Unlock() + if changeStream.resumeToken == nil { + return nil + } + var tokenCopy bson.Raw = *changeStream.resumeToken + return &tokenCopy +} + func constructChangeStreamPipeline(pipeline interface{}, options ChangeStreamOptions) interface{} { pipelinev := reflect.ValueOf(pipeline) From 950ed5a31db8164d0decb5a308948eb2141d636f Mon Sep 17 00:00:00 2001 From: "Artem V. Navrotskiy" Date: Thu, 21 Sep 2017 00:29:24 +0300 Subject: [PATCH 13/38] Introduce constants for BSON element types (#41) --- bson/bson.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/bson/bson.go b/bson/bson.go index ca1420825..ccc52a862 100644 --- a/bson/bson.go +++ b/bson/bson.go @@ -56,6 +56,39 @@ import ( // -------------------------------------------------------------------------- // The public API. +// Element types constants from BSON specification. +const ( + ElementFloat64 byte = 0x01 + ElementString byte = 0x02 + ElementDocument byte = 0x03 + ElementArray byte = 0x04 + ElementBinary byte = 0x05 + Element06 byte = 0x06 + ElementObjectId byte = 0x07 + ElementBool byte = 0x08 + ElementDatetime byte = 0x09 + ElementNil byte = 0x0A + ElementRegEx byte = 0x0B + ElementDBPointer byte = 0x0C + ElementJavaScriptWithoutScope byte = 0x0D + ElementSymbol byte = 0x0E + ElementJavaScriptWithScope byte = 0x0F + ElementInt32 byte = 0x10 + ElementTimestamp byte = 0x11 + ElementInt64 byte = 0x12 + ElementDecimal128 byte = 0x13 + ElementMinKey byte = 0xFF + ElementMaxKey byte = 0x7F + + BinaryGeneric byte = 0x00 + BinaryFunction byte = 0x01 + BinaryBinaryOld byte = 0x02 + BinaryUUIDOld byte = 0x03 + BinaryUUID byte = 0x04 + BinaryMD5 byte = 0x05 + BinaryUserDefined byte = 0x80 +) + // A value implementing the bson.Getter interface will have its GetBSON // method called when the given value has to be marshalled, and the result // of this method will be marshaled in place of the actual object. From d21a525152ad4b42e4aa8bb71a04bbbbf4e3c65a Mon Sep 17 00:00:00 2001 From: Georgy Date: Thu, 28 Sep 2017 17:57:06 +0300 Subject: [PATCH 14/38] bson.Unmarshal returns time in UTC (#42) --- bson/bson_test.go | 8 ++++---- bson/decode.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bson/bson_test.go b/bson/bson_test.go index 35bcc52f0..bf577901b 100644 --- a/bson/bson_test.go +++ b/bson/bson_test.go @@ -146,7 +146,7 @@ var allItems = []testItemType{ "\x08_\x00\x00"}, {bson.M{"_": true}, "\x08_\x00\x01"}, - {bson.M{"_": time.Unix(0, 258e6)}, // Note the NS <=> MS conversion. + {bson.M{"_": time.Unix(0, 258e6).UTC()}, // Note the NS <=> MS conversion. "\x09_\x00\x02\x01\x00\x00\x00\x00\x00\x00"}, {bson.M{"_": nil}, "\x0A_\x00"}, @@ -1265,7 +1265,7 @@ var twoWayCrossItems = []crossTypeItem{ {&condPtr{&falsevar}, map[string]bool{"v": false}}, {&condPtr{}, map[string]string{}}, - {&condTime{time.Unix(123456789, 123e6)}, map[string]time.Time{"v": time.Unix(123456789, 123e6)}}, + {&condTime{time.Unix(123456789, 123e6).UTC()}, map[string]time.Time{"v": time.Unix(123456789, 123e6).UTC()}}, {&condTime{}, map[string]string{}}, {&condStruct{struct{ A []int }{[]int{1}}}, bson.M{"v": bson.M{"a": []interface{}{1}}}}, @@ -1320,8 +1320,8 @@ var twoWayCrossItems = []crossTypeItem{ {&struct{ V time.Time }{}, map[string]interface{}{"v": time.Time{}}}, // zero time + 1 second + 1 millisecond; overflows int64 as nanoseconds - {&struct{ V time.Time }{time.Unix(-62135596799, 1e6).Local()}, - map[string]interface{}{"v": time.Unix(-62135596799, 1e6).Local()}}, + {&struct{ V time.Time }{time.Unix(-62135596799, 1e6).UTC()}, + map[string]interface{}{"v": time.Unix(-62135596799, 1e6).UTC()}}, // bson.D <=> []DocElem {&bson.D{{"a", bson.D{{"b", 1}, {"c", 2}}}}, &bson.D{{"a", bson.D{{"b", 1}, {"c", 2}}}}}, diff --git a/bson/decode.go b/bson/decode.go index 244eb3af1..417a4d14b 100644 --- a/bson/decode.go +++ b/bson/decode.go @@ -575,7 +575,7 @@ func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { if i == -62135596800000 { in = time.Time{} // In UTC for convenience. } else { - in = time.Unix(i/1e3, i%1e3*1e6) + in = time.Unix(i/1e3, i%1e3*1e6).UTC() } case 0x0A: // Nil in = nil From 9d743b415f5a310021d3cad11f52b6b6e7bc2846 Mon Sep 17 00:00:00 2001 From: Dom Dwyer Date: Thu, 28 Sep 2017 16:11:34 +0100 Subject: [PATCH 15/38] readme: add missing features / credit * Adds missing collation feature description (by @feliixx). * Adds missing 3.4 tests description (by @feliixx). * Adds BSON constants description (by @bozaro). * Adds UTC time.Time unmarshalling (by @gazoon). --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 349aaee43..14db4305d 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Further PR's (with tests) are welcome, but please maintain backwards compatibili * Hides SASL warnings ([details](https://github.com/globalsign/mgo/pull/7)) * Support for partial indexes ([detials](https://github.com/domodwyer/mgo/commit/5efe8eccb028238d93c222828cae4806aeae9f51)) * Fixes timezone handling ([details](https://github.com/go-mgo/mgo/pull/464)) -* Integration tests run against newest MongoDB 3.2 releases ([details](https://github.com/globalsign/mgo/pull/4), [more](https://github.com/globalsign/mgo/pull/24)) +* Integration tests run against MongoDB 3.2 & 3.4 releases ([details](https://github.com/globalsign/mgo/pull/4), [more](https://github.com/globalsign/mgo/pull/24), [more](https://github.com/globalsign/mgo/pull/35)) * Improved multi-document transaction performance ([details](https://github.com/globalsign/mgo/pull/10), [more](https://github.com/globalsign/mgo/pull/11), [more](https://github.com/globalsign/mgo/pull/16)) * Fixes cursor timeouts ([details](https://jira.mongodb.org/browse/SERVER-24899)) * Support index hints and timeouts for count queries ([details](https://github.com/globalsign/mgo/pull/17)) @@ -25,10 +25,15 @@ Further PR's (with tests) are welcome, but please maintain backwards compatibili * Supports dropping all indexes on a collection ([details](https://github.com/globalsign/mgo/pull/25)) * Annotates log entries/profiler output with optional appName on 3.4+ ([details](https://github.com/globalsign/mgo/pull/28)) * Support for read-only [views](https://docs.mongodb.com/manual/core/views/) in 3.4+ ([details](https://github.com/globalsign/mgo/pull/33)) +* Support for [collations](https://docs.mongodb.com/manual/reference/collation/) in 3.4+ ([details](https://github.com/globalsign/mgo/pull/37)) +* Provide BSON constants for convenience/sanity ([details](https://github.com/globalsign/mgo/pull/41)) +* Consistently unmarshal time.Time values as UTC ([details](https://github.com/globalsign/mgo/pull/42)) + --- ### Thanks to +* @bozaro * @BenLubar * @carter2000 * @cezarsa @@ -37,6 +42,7 @@ Further PR's (with tests) are welcome, but please maintain backwards compatibili * @feliixx * @fmpwizard * @jameinel +* @gazoon * @mapete94 * @Reenjii * @smoya From 97bd0cddadb6fed3fff2860d3f0748d7407b2bde Mon Sep 17 00:00:00 2001 From: Adrien Petel Date: Fri, 6 Oct 2017 13:38:25 +0400 Subject: [PATCH 16/38] fix golint, go vet and gofmt warnings (#44) Fixes #43 --- .travis.yml | 3 + auth.go | 2 +- auth_test.go | 10 +- bson/bson.go | 32 +-- bson/bson_test.go | 176 +++++++------- bson/decimal.go | 2 + bson/decimal_test.go | 238 +++++++++---------- bson/decode.go | 4 +- bson/encode.go | 2 +- bson/json_test.go | 2 +- bulk_test.go | 6 +- cluster.go | 3 +- cluster_test.go | 17 +- dbtest/dbserver.go | 2 +- dbtest/dbserver_test.go | 2 + gridfs.go | 51 ++-- internal/json/decode.go | 2 +- internal/json/decode_test.go | 2 +- internal/json/encode.go | 6 +- internal/json/stream_test.go | 2 +- internal/sasl/sasl.go | 6 +- internal/scram/scram.go | 6 +- log.go | 12 +- server.go | 3 +- session.go | 441 +++++++++++++++++++++-------------- session_test.go | 98 ++++---- socket.go | 24 +- stats.go | 10 + txn/debug.go | 10 +- txn/flusher.go | 143 ++++++------ txn/txn.go | 35 +-- txn/txn_test.go | 23 +- 32 files changed, 764 insertions(+), 611 deletions(-) diff --git a/.travis.yml b/.travis.yml index 430844718..8d4428d1f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,8 +33,11 @@ install: - go get gopkg.in/check.v1 - go get gopkg.in/yaml.v2 - go get gopkg.in/tomb.v2 + - go get github.com/golang/lint/golint before_script: + - golint ./... | grep -v 'ID' | cat + - go vet github.com/globalsign/mgo/bson github.com/globalsign/mgo/txn github.com/globalsign/mgo - export NOIPV6=1 - make startdb diff --git a/auth.go b/auth.go index 388e62105..75d2ebc36 100644 --- a/auth.go +++ b/auth.go @@ -61,7 +61,7 @@ type getNonceCmd struct { type getNonceResult struct { Nonce string - Err string "$err" + Err string `bson:"$err"` Code int } diff --git a/auth_test.go b/auth_test.go index 9f04b2246..ed1af5abf 100644 --- a/auth_test.go +++ b/auth_test.go @@ -580,7 +580,7 @@ func (s *S) TestAuthLoginCachingWithNewSession(c *C) { } func (s *S) TestAuthLoginCachingAcrossPool(c *C) { - // Logins are cached even when the conenction goes back + // Logins are cached even when the connection goes back // into the pool. session, err := mgo.Dial("localhost:40002") @@ -934,7 +934,7 @@ func (s *S) TestAuthX509Cred(c *C) { x509Subject := "CN=localhost,OU=Client,O=MGO,L=MGO,ST=MGO,C=GO" externalDB := session.DB("$external") - var x509User mgo.User = mgo.User{ + var x509User = mgo.User{ Username: x509Subject, OtherDBRoles: map[string][]mgo.Role{"admin": {mgo.RoleRoot}}, } @@ -1080,11 +1080,11 @@ func (kerberosSuite *KerberosSuite) TestAuthKerberosURL(c *C) { c.Skip("no -kerberos") } c.Logf("Connecting to %s...", kerberosHost) - connectUri := url.QueryEscape(kerberosUser) + "@" + kerberosHost + "?authMechanism=GSSAPI" + connectURI := url.QueryEscape(kerberosUser) + "@" + kerberosHost + "?authMechanism=GSSAPI" if runtime.GOOS == "windows" { - connectUri = url.QueryEscape(kerberosUser) + ":" + url.QueryEscape(getWindowsKerberosPassword()) + "@" + kerberosHost + "?authMechanism=GSSAPI" + connectURI = url.QueryEscape(kerberosUser) + ":" + url.QueryEscape(getWindowsKerberosPassword()) + "@" + kerberosHost + "?authMechanism=GSSAPI" } - session, err := mgo.Dial(connectUri) + session, err := mgo.Dial(connectURI) c.Assert(err, IsNil) defer session.Close() n, err := session.DB("kerberos").C("test").Find(M{}).Count() diff --git a/bson/bson.go b/bson/bson.go index ccc52a862..d960f7a37 100644 --- a/bson/bson.go +++ b/bson/bson.go @@ -89,7 +89,7 @@ const ( BinaryUserDefined byte = 0x80 ) -// A value implementing the bson.Getter interface will have its GetBSON +// Getter interface: a value implementing the bson.Getter interface will have its GetBSON // method called when the given value has to be marshalled, and the result // of this method will be marshaled in place of the actual object. // @@ -99,12 +99,12 @@ type Getter interface { GetBSON() (interface{}, error) } -// A value implementing the bson.Setter interface will receive the BSON +// Setter interface: a value implementing the bson.Setter interface will receive the BSON // value via the SetBSON method during unmarshaling, and the object // itself will not be changed as usual. // // If setting the value works, the method should return nil or alternatively -// bson.SetZero to set the respective field to its zero value (nil for +// bson.ErrSetZero to set the respective field to its zero value (nil for // pointer types). If SetBSON returns a value of type bson.TypeError, the // BSON value will be omitted from a map or slice being decoded and the // unmarshalling will continue. If it returns any other non-nil error, the @@ -130,10 +130,10 @@ type Setter interface { SetBSON(raw Raw) error } -// SetZero may be returned from a SetBSON method to have the value set to +// ErrSetZero may be returned from a SetBSON method to have the value set to // its respective zero value. When used in pointer values, this will set the // field to nil rather than to the pre-allocated value. -var SetZero = errors.New("set to zero") +var ErrSetZero = errors.New("set to zero") // M is a convenient alias for a map[string]interface{} map, useful for // dealing with BSON in a native way. For instance: @@ -189,7 +189,7 @@ type Raw struct { // documents in general. type RawD []RawDocElem -// See the RawD type. +// RawDocElem elements of RawD type. type RawDocElem struct { Name string Value Raw @@ -199,7 +199,7 @@ type RawDocElem struct { // long. MongoDB objects by default have such a property set in their "_id" // property. // -// http://www.mongodb.org/display/DOCS/Object+IDs +// http://www.mongodb.org/display/DOCS/Object+Ids type ObjectId string // ObjectIdHex returns an ObjectId from the provided hex representation. @@ -225,7 +225,7 @@ func IsObjectIdHex(s string) bool { // objectIdCounter is atomically incremented when generating a new ObjectId // using NewObjectId() function. It's used as a counter part of an id. -var objectIdCounter uint32 = readRandomUint32() +var objectIdCounter = readRandomUint32() // readRandomUint32 returns a random objectIdCounter. func readRandomUint32() uint32 { @@ -333,12 +333,12 @@ func (id *ObjectId) UnmarshalJSON(data []byte) error { return nil } if len(data) != 26 || data[0] != '"' || data[25] != '"' { - return errors.New(fmt.Sprintf("invalid ObjectId in JSON: %s", string(data))) + return fmt.Errorf("invalid ObjectId in JSON: %s", string(data)) } var buf [12]byte _, err := hex.Decode(buf[:], data[1:25]) if err != nil { - return errors.New(fmt.Sprintf("invalid ObjectId in JSON: %s (%s)", string(data), err)) + return fmt.Errorf("invalid ObjectId in JSON: %s (%s)", string(data), err) } *id = ObjectId(string(buf[:])) return nil @@ -604,12 +604,12 @@ func Unmarshal(in []byte, out interface{}) (err error) { d := newDecoder(in) d.readDocTo(v) if d.i < len(d.in) { - return errors.New("Document is corrupted") + return errors.New("document is corrupted") } case reflect.Struct: - return errors.New("Unmarshal can't deal with struct values. Use a pointer.") + return errors.New("unmarshal can't deal with struct values. Use a pointer") default: - return errors.New("Unmarshal needs a map or a pointer to a struct.") + return errors.New("unmarshal needs a map or a pointer to a struct") } return nil } @@ -633,13 +633,15 @@ func (raw Raw) Unmarshal(out interface{}) (err error) { return &TypeError{v.Type(), raw.Kind} } case reflect.Struct: - return errors.New("Raw Unmarshal can't deal with struct values. Use a pointer.") + return errors.New("raw Unmarshal can't deal with struct values. Use a pointer") default: - return errors.New("Raw Unmarshal needs a map or a valid pointer.") + return errors.New("raw Unmarshal needs a map or a valid pointer") } return nil } +// TypeError store details for type error occuring +// during unmarshaling type TypeError struct { Type reflect.Type Kind byte diff --git a/bson/bson_test.go b/bson/bson_test.go index bf577901b..6b8c0cd0b 100644 --- a/bson/bson_test.go +++ b/bson/bson_test.go @@ -134,13 +134,13 @@ var allItems = []testItemType{ "\x04_\x00\r\x00\x00\x00\x080\x00\x01\x081\x00\x00\x00"}, {bson.M{"_": []byte("yo")}, "\x05_\x00\x02\x00\x00\x00\x00yo"}, - {bson.M{"_": bson.Binary{0x80, []byte("udef")}}, + {bson.M{"_": bson.Binary{Kind: 0x80, Data: []byte("udef")}}, "\x05_\x00\x04\x00\x00\x00\x80udef"}, {bson.M{"_": bson.Undefined}, // Obsolete, but still seen in the wild. "\x06_\x00"}, {bson.M{"_": bson.ObjectId("0123456789ab")}, "\x07_\x000123456789ab"}, - {bson.M{"_": bson.DBPointer{"testnamespace", bson.ObjectId("0123456789ab")}}, + {bson.M{"_": bson.DBPointer{Namespace: "testnamespace", Id: bson.ObjectId("0123456789ab")}}, "\x0C_\x00\x0e\x00\x00\x00testnamespace\x000123456789ab"}, {bson.M{"_": false}, "\x08_\x00\x00"}, @@ -150,13 +150,13 @@ var allItems = []testItemType{ "\x09_\x00\x02\x01\x00\x00\x00\x00\x00\x00"}, {bson.M{"_": nil}, "\x0A_\x00"}, - {bson.M{"_": bson.RegEx{"ab", "cd"}}, + {bson.M{"_": bson.RegEx{Pattern: "ab", Options: "cd"}}, "\x0B_\x00ab\x00cd\x00"}, - {bson.M{"_": bson.JavaScript{"code", nil}}, + {bson.M{"_": bson.JavaScript{Code: "code", Scope: nil}}, "\x0D_\x00\x05\x00\x00\x00code\x00"}, {bson.M{"_": bson.Symbol("sym")}, "\x0E_\x00\x04\x00\x00\x00sym\x00"}, - {bson.M{"_": bson.JavaScript{"code", bson.M{"": nil}}}, + {bson.M{"_": bson.JavaScript{Code: "code", Scope: bson.M{"": nil}}}, "\x0F_\x00\x14\x00\x00\x00\x05\x00\x00\x00code\x00" + "\x07\x00\x00\x00\x0A\x00\x00"}, {bson.M{"_": 258}, @@ -200,7 +200,7 @@ func (s *S) TestUnmarshalRawAllItems(c *C) { continue } pv := reflect.New(reflect.ValueOf(value).Type()) - raw := bson.Raw{item.data[0], []byte(item.data[3:])} + raw := bson.Raw{Kind: item.data[0], Data: []byte(item.data[3:])} c.Logf("Unmarshal raw: %#v, %#v", raw, pv.Interface()) err := raw.Unmarshal(pv.Interface()) c.Assert(err, IsNil) @@ -209,7 +209,7 @@ func (s *S) TestUnmarshalRawAllItems(c *C) { } func (s *S) TestUnmarshalRawIncompatible(c *C) { - raw := bson.Raw{0x08, []byte{0x01}} // true + raw := bson.Raw{Kind: 0x08, Data: []byte{0x01}} // true err := raw.Unmarshal(&struct{}{}) c.Assert(err, ErrorMatches, "BSON kind 0x08 isn't compatible with type struct \\{\\}") } @@ -258,15 +258,15 @@ func (s *S) TestMarshalBuffer(c *C) { var oneWayMarshalItems = []testItemType{ // These are being passed as pointers, and will unmarshal as values. - {bson.M{"": &bson.Binary{0x02, []byte("old")}}, + {bson.M{"": &bson.Binary{Kind: 0x02, Data: []byte("old")}}, "\x05\x00\x07\x00\x00\x00\x02\x03\x00\x00\x00old"}, - {bson.M{"": &bson.Binary{0x80, []byte("udef")}}, + {bson.M{"": &bson.Binary{Kind: 0x80, Data: []byte("udef")}}, "\x05\x00\x04\x00\x00\x00\x80udef"}, - {bson.M{"": &bson.RegEx{"ab", "cd"}}, + {bson.M{"": &bson.RegEx{Pattern: "ab", Options: "cd"}}, "\x0B\x00ab\x00cd\x00"}, - {bson.M{"": &bson.JavaScript{"code", nil}}, + {bson.M{"": &bson.JavaScript{Code: "code", Scope: nil}}, "\x0D\x00\x05\x00\x00\x00code\x00"}, - {bson.M{"": &bson.JavaScript{"code", bson.M{"": nil}}}, + {bson.M{"": &bson.JavaScript{Code: "code", Scope: bson.M{"": nil}}}, "\x0F\x00\x14\x00\x00\x00\x05\x00\x00\x00code\x00" + "\x07\x00\x00\x00\x0A\x00\x00"}, @@ -283,9 +283,9 @@ var oneWayMarshalItems = []testItemType{ "\x04\x00\r\x00\x00\x00\x080\x00\x01\x081\x00\x00\x00"}, // Will unmarshal as a []byte. - {bson.M{"": bson.Binary{0x00, []byte("yo")}}, + {bson.M{"": bson.Binary{Kind: 0x00, Data: []byte("yo")}}, "\x05\x00\x02\x00\x00\x00\x00yo"}, - {bson.M{"": bson.Binary{0x02, []byte("old")}}, + {bson.M{"": bson.Binary{Kind: 0x02, Data: []byte("old")}}, "\x05\x00\x07\x00\x00\x00\x02\x03\x00\x00\x00old"}, // No way to preserve the type information here. We might encode as a zero @@ -338,7 +338,7 @@ type specSample1 struct { } type specSample2 struct { - BSON []interface{} "BSON" + BSON []interface{} `bson:"BSON"` } var structSampleItems = []testItemType{ @@ -396,7 +396,7 @@ var structItems = []testItemType{ {&struct{ Byte byte }{0}, "\x10byte\x00\x00\x00\x00\x00"}, {&struct { - V byte "Tag" + V byte `bson:"Tag"` }{8}, "\x10Tag\x00\x08\x00\x00\x00"}, {&struct { @@ -411,9 +411,9 @@ var structItems = []testItemType{ {&struct{ A, C, B, D, F, E *byte }{}, "\x0Aa\x00\x0Ac\x00\x0Ab\x00\x0Ad\x00\x0Af\x00\x0Ae\x00"}, - {&struct{ V bson.Raw }{bson.Raw{0x03, []byte("\x0f\x00\x00\x00\x10byte\x00\b\x00\x00\x00\x00")}}, + {&struct{ V bson.Raw }{bson.Raw{Kind: 0x03, Data: []byte("\x0f\x00\x00\x00\x10byte\x00\b\x00\x00\x00\x00")}}, "\x03v\x00" + "\x0f\x00\x00\x00\x10byte\x00\b\x00\x00\x00\x00"}, - {&struct{ V bson.Raw }{bson.Raw{0x10, []byte("\x00\x00\x00\x00")}}, + {&struct{ V bson.Raw }{bson.Raw{Kind: 0x10, Data: []byte("\x00\x00\x00\x00")}}, "\x10v\x00" + "\x00\x00\x00\x00"}, // Byte arrays. @@ -438,7 +438,7 @@ func (s *S) TestUnmarshalStructItems(c *C) { func (s *S) TestUnmarshalRawStructItems(c *C) { for i, item := range structItems { - raw := bson.Raw{0x03, []byte(wrapInDoc(item.data))} + raw := bson.Raw{Kind: 0x03, Data: []byte(wrapInDoc(item.data))} zero := makeZeroDoc(item.obj) err := raw.Unmarshal(zero) c.Assert(err, IsNil) @@ -449,7 +449,7 @@ func (s *S) TestUnmarshalRawStructItems(c *C) { func (s *S) TestUnmarshalRawNil(c *C) { // Regression test: shouldn't try to nil out the pointer itself, // as it's not settable. - raw := bson.Raw{0x0A, []byte{}} + raw := bson.Raw{Kind: 0x0A, Data: []byte{}} err := raw.Unmarshal(&struct{}{}) c.Assert(err, IsNil) } @@ -469,25 +469,25 @@ type ignoreField struct { var marshalItems = []testItemType{ // Ordered document dump. Will unmarshal as a dictionary by default. - {bson.D{{"a", nil}, {"c", nil}, {"b", nil}, {"d", nil}, {"f", nil}, {"e", true}}, + {bson.D{{Name: "a", Value: nil}, {Name: "c", Value: nil}, {Name: "b", Value: nil}, {Name: "d", Value: nil}, {Name: "f", Value: nil}, {Name: "e", Value: true}}, "\x0Aa\x00\x0Ac\x00\x0Ab\x00\x0Ad\x00\x0Af\x00\x08e\x00\x01"}, - {MyD{{"a", nil}, {"c", nil}, {"b", nil}, {"d", nil}, {"f", nil}, {"e", true}}, + {MyD{{Name: "a", Value: nil}, {Name: "c", Value: nil}, {Name: "b", Value: nil}, {Name: "d", Value: nil}, {Name: "f", Value: nil}, {Name: "e", Value: true}}, "\x0Aa\x00\x0Ac\x00\x0Ab\x00\x0Ad\x00\x0Af\x00\x08e\x00\x01"}, - {&dOnIface{bson.D{{"a", nil}, {"c", nil}, {"b", nil}, {"d", true}}}, + {&dOnIface{bson.D{{Name: "a", Value: nil}, {Name: "c", Value: nil}, {Name: "b", Value: nil}, {Name: "d", Value: true}}}, "\x03d\x00" + wrapInDoc("\x0Aa\x00\x0Ac\x00\x0Ab\x00\x08d\x00\x01")}, - {bson.RawD{{"a", bson.Raw{0x0A, nil}}, {"c", bson.Raw{0x0A, nil}}, {"b", bson.Raw{0x08, []byte{0x01}}}}, + {bson.RawD{{Name: "a", Value: bson.Raw{Kind: 0x0A, Data: nil}}, {Name: "c", Value: bson.Raw{Kind: 0x0A, Data: nil}}, {Name: "b", Value: bson.Raw{Kind: 0x08, Data: []byte{0x01}}}}, "\x0Aa\x00" + "\x0Ac\x00" + "\x08b\x00\x01"}, - {MyRawD{{"a", bson.Raw{0x0A, nil}}, {"c", bson.Raw{0x0A, nil}}, {"b", bson.Raw{0x08, []byte{0x01}}}}, + {MyRawD{{Name: "a", Value: bson.Raw{Kind: 0x0A, Data: nil}}, {Name: "c", Value: bson.Raw{Kind: 0x0A, Data: nil}}, {Name: "b", Value: bson.Raw{Kind: 0x08, Data: []byte{0x01}}}}, "\x0Aa\x00" + "\x0Ac\x00" + "\x08b\x00\x01"}, - {&dOnIface{bson.RawD{{"a", bson.Raw{0x0A, nil}}, {"c", bson.Raw{0x0A, nil}}, {"b", bson.Raw{0x08, []byte{0x01}}}}}, + {&dOnIface{bson.RawD{{Name: "a", Value: bson.Raw{Kind: 0x0A, Data: nil}}, {Name: "c", Value: bson.Raw{Kind: 0x0A, Data: nil}}, {Name: "b", Value: bson.Raw{Kind: 0x08, Data: []byte{0x01}}}}}, "\x03d\x00" + wrapInDoc("\x0Aa\x00"+"\x0Ac\x00"+"\x08b\x00\x01")}, {&ignoreField{"before", "ignore", "after"}, "\x02before\x00\a\x00\x00\x00before\x00\x02after\x00\x06\x00\x00\x00after\x00"}, // Marshalling a Raw document does nothing. - {bson.Raw{0x03, []byte(wrapInDoc("anything"))}, + {bson.Raw{Kind: 0x03, Data: []byte(wrapInDoc("anything"))}, "anything"}, {bson.Raw{Data: []byte(wrapInDoc("anything"))}, "anything"}, @@ -536,15 +536,15 @@ var unmarshalItems = []testItemType{ "\x02str\x00\x02\x00\x00\x00s\x00"}, // Ordered document. - {&struct{ bson.D }{bson.D{{"a", nil}, {"c", nil}, {"b", nil}, {"d", true}}}, + {&struct{ bson.D }{bson.D{{Name: "a", Value: nil}, {Name: "c", Value: nil}, {Name: "b", Value: nil}, {Name: "d", Value: true}}}, "\x03d\x00" + wrapInDoc("\x0Aa\x00\x0Ac\x00\x0Ab\x00\x08d\x00\x01")}, // Raw document. - {&bson.Raw{0x03, []byte(wrapInDoc("\x10byte\x00\x08\x00\x00\x00"))}, + {&bson.Raw{Kind: 0x03, Data: []byte(wrapInDoc("\x10byte\x00\x08\x00\x00\x00"))}, "\x10byte\x00\x08\x00\x00\x00"}, // RawD document. - {&struct{ bson.RawD }{bson.RawD{{"a", bson.Raw{0x0A, []byte{}}}, {"c", bson.Raw{0x0A, []byte{}}}, {"b", bson.Raw{0x08, []byte{0x01}}}}}, + {&struct{ bson.RawD }{bson.RawD{{Name: "a", Value: bson.Raw{Kind: 0x0A, Data: []byte{}}}, {Name: "c", Value: bson.Raw{Kind: 0x0A, Data: []byte{}}}, {Name: "b", Value: bson.Raw{Kind: 0x08, Data: []byte{0x01}}}}}, "\x03rawd\x00" + wrapInDoc("\x0Aa\x00\x0Ac\x00\x08b\x00\x01")}, // Decode old binary. @@ -580,7 +580,7 @@ func (s *S) TestUnmarshalNilInStruct(c *C) { type structWithDupKeys struct { Name byte - Other byte "name" // Tag should precede. + Other byte `bson:"name"` // Tag should precede. } var marshalErrorItems = []testItemType{ @@ -594,11 +594,11 @@ var marshalErrorItems = []testItemType{ "Can't marshal complex128 in a BSON document"}, {&structWithDupKeys{}, "Duplicated key 'name' in struct bson_test.structWithDupKeys"}, - {bson.Raw{0xA, []byte{}}, + {bson.Raw{Kind: 0xA, Data: []byte{}}, "Attempted to marshal Raw kind 10 as a document"}, - {bson.Raw{0x3, []byte{}}, + {bson.Raw{Kind: 0x3, Data: []byte{}}, "Attempted to marshal empty Raw document"}, - {bson.M{"w": bson.Raw{0x3, []byte{}}}, + {bson.M{"w": bson.Raw{Kind: 0x3, Data: []byte{}}}, "Attempted to marshal empty Raw document"}, {&inlineCantPtr{&struct{ A, B int }{1, 2}}, "Option ,inline needs a struct value or map field"}, @@ -646,11 +646,11 @@ var unmarshalErrorItems = []unmarshalErrorType{ {struct{ Name bool }{}, "\x10name\x00\x08\x00\x00\x00", - "Unmarshal can't deal with struct values. Use a pointer."}, + "unmarshal can't deal with struct values. Use a pointer"}, {123, "\x10name\x00\x08\x00\x00\x00", - "Unmarshal needs a map or a pointer to a struct."}, + "unmarshal needs a map or a pointer to a struct"}, {nil, "\x08\x62\x00\x02", @@ -683,20 +683,20 @@ type unmarshalRawErrorType struct { var unmarshalRawErrorItems = []unmarshalRawErrorType{ // Tag name conflicts with existing parameter. {&structWithDupKeys{}, - bson.Raw{0x03, []byte("\x10byte\x00\x08\x00\x00\x00")}, + bson.Raw{Kind: 0x03, Data: []byte("\x10byte\x00\x08\x00\x00\x00")}, "Duplicated key 'name' in struct bson_test.structWithDupKeys"}, {&struct{}{}, - bson.Raw{0xEE, []byte{}}, + bson.Raw{Kind: 0xEE, Data: []byte{}}, "Unknown element kind \\(0xEE\\)"}, {struct{ Name bool }{}, - bson.Raw{0x10, []byte("\x08\x00\x00\x00")}, - "Raw Unmarshal can't deal with struct values. Use a pointer."}, + bson.Raw{Kind: 0x10, Data: []byte("\x08\x00\x00\x00")}, + "raw Unmarshal can't deal with struct values. Use a pointer"}, {123, - bson.Raw{0x10, []byte("\x08\x00\x00\x00")}, - "Raw Unmarshal needs a map or a valid pointer."}, + bson.Raw{Kind: 0x10, Data: []byte("\x08\x00\x00\x00")}, + "raw Unmarshal needs a map or a valid pointer"}, } func (s *S) TestUnmarshalRawErrorItems(c *C) { @@ -768,11 +768,11 @@ func (o *setterType) SetBSON(raw bson.Raw) error { } type ptrSetterDoc struct { - Field *setterType "_" + Field *setterType `bson:"_"` } type valSetterDoc struct { - Field setterType "_" + Field setterType `bson:"_"` } func (s *S) TestUnmarshalAllItemsWithPtrSetter(c *C) { @@ -856,12 +856,12 @@ func (s *S) TestUnmarshalSetterErrors(c *C) { } func (s *S) TestDMap(c *C) { - d := bson.D{{"a", 1}, {"b", 2}} + d := bson.D{{Name: "a", Value: 1}, {Name: "b", Value: 2}} c.Assert(d.Map(), DeepEquals, bson.M{"a": 1, "b": 2}) } -func (s *S) TestUnmarshalSetterSetZero(c *C) { - setterResult["foo"] = bson.SetZero +func (s *S) TestUnmarshalSetterErrSetZero(c *C) { + setterResult["foo"] = bson.ErrSetZero defer delete(setterResult, "field") data, err := bson.Marshal(bson.M{"field": "foo"}) @@ -892,7 +892,7 @@ func (t *typeWithGetter) GetBSON() (interface{}, error) { } type docWithGetterField struct { - Field *typeWithGetter "_" + Field *typeWithGetter `bson:"_"` } func (s *S) TestMarshalAllItemsWithGetter(c *C) { @@ -938,7 +938,7 @@ func (t intGetter) GetBSON() (interface{}, error) { } type typeWithIntGetter struct { - V intGetter ",minsize" + V intGetter `bson:",minsize"` } func (s *S) TestMarshalShortWithGetter(c *C) { @@ -970,96 +970,96 @@ type crossTypeItem struct { } type condStr struct { - V string ",omitempty" + V string `bson:",omitempty"` } type condStrNS struct { V string `a:"A" bson:",omitempty" b:"B"` } type condBool struct { - V bool ",omitempty" + V bool `bson:",omitempty"` } type condInt struct { - V int ",omitempty" + V int `bson:",omitempty"` } type condUInt struct { - V uint ",omitempty" + V uint `bson:",omitempty"` } type condFloat struct { - V float64 ",omitempty" + V float64 `bson:",omitempty"` } type condIface struct { - V interface{} ",omitempty" + V interface{} `bson:",omitempty"` } type condPtr struct { - V *bool ",omitempty" + V *bool `bson:",omitempty"` } type condSlice struct { - V []string ",omitempty" + V []string `bson:",omitempty"` } type condMap struct { - V map[string]int ",omitempty" + V map[string]int `bson:",omitempty"` } type namedCondStr struct { - V string "myv,omitempty" + V string `bson:"myv,omitempty"` } type condTime struct { - V time.Time ",omitempty" + V time.Time `bson:",omitempty"` } type condStruct struct { - V struct{ A []int } ",omitempty" + V struct{ A []int } `bson:",omitempty"` } type condRaw struct { - V bson.Raw ",omitempty" + V bson.Raw `bson:",omitempty"` } type shortInt struct { - V int64 ",minsize" + V int64 `bson:",minsize"` } type shortUint struct { - V uint64 ",minsize" + V uint64 `bson:",minsize"` } type shortIface struct { - V interface{} ",minsize" + V interface{} `bson:",minsize"` } type shortPtr struct { - V *int64 ",minsize" + V *int64 `bson:",minsize"` } type shortNonEmptyInt struct { - V int64 ",minsize,omitempty" + V int64 `bson:",minsize,omitempty"` } type inlineInt struct { - V struct{ A, B int } ",inline" + V struct{ A, B int } `bson:",inline"` } type inlineCantPtr struct { - V *struct{ A, B int } ",inline" + V *struct{ A, B int } `bson:",inline"` } type inlineDupName struct { A int - V struct{ A, B int } ",inline" + V struct{ A, B int } `bson:",inline"` } type inlineMap struct { A int - M map[string]interface{} ",inline" + M map[string]interface{} `bson:",inline"` } type inlineMapInt struct { A int - M map[string]int ",inline" + M map[string]int `bson:",inline"` } type inlineMapMyM struct { A int - M MyM ",inline" + M MyM `bson:",inline"` } type inlineDupMap struct { - M1 map[string]interface{} ",inline" - M2 map[string]interface{} ",inline" + M1 map[string]interface{} `bson:",inline"` + M2 map[string]interface{} `bson:",inline"` } type inlineBadKeyMap struct { - M map[int]int ",inline" + M map[int]int `bson:",inline"` } type inlineUnexported struct { - M map[string]interface{} ",inline" - unexported ",inline" + M map[string]interface{} `bson:",inline"` + unexported `bson:",inline"` } type unexported struct { A int @@ -1077,7 +1077,7 @@ func (s getterSetterD) GetBSON() (interface{}, error) { func (s *getterSetterD) SetBSON(raw bson.Raw) error { var doc bson.D err := raw.Unmarshal(&doc) - doc = append(doc, bson.DocElem{"suffix", true}) + doc = append(doc, bson.DocElem{Name: "suffix", Value: true}) *s = getterSetterD(doc) return err } @@ -1085,7 +1085,7 @@ func (s *getterSetterD) SetBSON(raw bson.Raw) error { type getterSetterInt int func (i getterSetterInt) GetBSON() (interface{}, error) { - return bson.D{{"a", int(i)}}, nil + return bson.D{{Name: "a", Value: int(i)}}, nil } func (i *getterSetterInt) SetBSON(raw bson.Raw) error { @@ -1324,13 +1324,13 @@ var twoWayCrossItems = []crossTypeItem{ map[string]interface{}{"v": time.Unix(-62135596799, 1e6).UTC()}}, // bson.D <=> []DocElem - {&bson.D{{"a", bson.D{{"b", 1}, {"c", 2}}}}, &bson.D{{"a", bson.D{{"b", 1}, {"c", 2}}}}}, - {&bson.D{{"a", bson.D{{"b", 1}, {"c", 2}}}}, &MyD{{"a", MyD{{"b", 1}, {"c", 2}}}}}, - {&struct{ V MyD }{MyD{{"a", 1}}}, &bson.D{{"v", bson.D{{"a", 1}}}}}, + {&bson.D{{Name: "a", Value: bson.D{{Name: "b", Value: 1}, {Name: "c", Value: 2}}}}, &bson.D{{Name: "a", Value: bson.D{{Name: "b", Value: 1}, {Name: "c", Value: 2}}}}}, + {&bson.D{{Name: "a", Value: bson.D{{Name: "b", Value: 1}, {Name: "c", Value: 2}}}}, &MyD{{Name: "a", Value: MyD{{Name: "b", Value: 1}, {Name: "c", Value: 2}}}}}, + {&struct{ V MyD }{MyD{{Name: "a", Value: 1}}}, &bson.D{{Name: "v", Value: bson.D{{Name: "a", Value: 1}}}}}, // bson.RawD <=> []RawDocElem - {&bson.RawD{{"a", bson.Raw{0x08, []byte{0x01}}}}, &bson.RawD{{"a", bson.Raw{0x08, []byte{0x01}}}}}, - {&bson.RawD{{"a", bson.Raw{0x08, []byte{0x01}}}}, &MyRawD{{"a", bson.Raw{0x08, []byte{0x01}}}}}, + {&bson.RawD{{Name: "a", Value: bson.Raw{Kind: 0x08, Data: []byte{0x01}}}}, &bson.RawD{{Name: "a", Value: bson.Raw{Kind: 0x08, Data: []byte{0x01}}}}}, + {&bson.RawD{{Name: "a", Value: bson.Raw{Kind: 0x08, Data: []byte{0x01}}}}, &MyRawD{{Name: "a", Value: bson.Raw{Kind: 0x08, Data: []byte{0x01}}}}}, // bson.M <=> map {bson.M{"a": bson.M{"b": 1, "c": 2}}, MyM{"a": MyM{"b": 1, "c": 2}}}, @@ -1345,8 +1345,8 @@ var twoWayCrossItems = []crossTypeItem{ {&struct{ N json.Number }{"9223372036854776000"}, map[string]interface{}{"n": float64(1 << 63)}}, // bson.D <=> non-struct getter/setter - {&bson.D{{"a", 1}}, &getterSetterD{{"a", 1}, {"suffix", true}}}, - {&bson.D{{"a", 42}}, &gsintvar}, + {&bson.D{{Name: "a", Value: 1}}, &getterSetterD{{Name: "a", Value: 1}, {Name: "suffix", Value: true}}}, + {&bson.D{{Name: "a", Value: 42}}, &gsintvar}, // Interface slice setter. {&struct{ V ifaceSlice }{ifaceSlice{nil, nil, nil}}, bson.M{"v": []interface{}{3}}}, @@ -1368,7 +1368,7 @@ var oneWayCrossItems = []crossTypeItem{ // Ensure omitempty on struct with private fields works properly. {&struct { - V struct{ v time.Time } ",omitempty" + V struct{ v time.Time } `bson:",omitempty"` }{}, map[string]interface{}{}}, // Attempt to marshal slice into RawD (issue #120). diff --git a/bson/decimal.go b/bson/decimal.go index 3d2f70020..672ba1825 100644 --- a/bson/decimal.go +++ b/bson/decimal.go @@ -144,6 +144,8 @@ func dErr(s string) (Decimal128, error) { return dNaN, fmt.Errorf("cannot parse %q as a decimal128", s) } +// ParseDecimal128 parse a string and return the corresponding value as +// a decimal128 func ParseDecimal128(s string) (Decimal128, error) { orig := s if s == "" { diff --git a/bson/decimal_test.go b/bson/decimal_test.go index 2cc510cca..142adc4a4 100644 --- a/bson/decimal_test.go +++ b/bson/decimal_test.go @@ -27,162 +27,162 @@ package bson_test import ( - "encoding/hex" - "encoding/json" - "fmt" - "regexp" - "strings" + "encoding/hex" + "encoding/json" + "fmt" + "regexp" + "strings" - "github.com/globalsign/mgo/bson" + "github.com/globalsign/mgo/bson" - . "gopkg.in/check.v1" + . "gopkg.in/check.v1" ) // -------------------------------------------------------------------------- // Decimal tests type decimalTests struct { - Valid []struct { - Description string `json:"description"` - BSON string `json:"bson"` - CanonicalBSON string `json:"canonical_bson"` - ExtJSON string `json:"extjson"` - CanonicalExtJSON string `json:"canonical_extjson"` - Lossy bool `json:"lossy"` - } `json:"valid"` + Valid []struct { + Description string `json:"description"` + BSON string `json:"bson"` + CanonicalBSON string `json:"canonical_bson"` + ExtJSON string `json:"extjson"` + CanonicalExtJSON string `json:"canonical_extjson"` + Lossy bool `json:"lossy"` + } `json:"valid"` - ParseErrors []struct { - Description string `json:"description"` - String string `json:"string"` - } `json:"parseErrors"` + ParseErrors []struct { + Description string `json:"description"` + String string `json:"string"` + } `json:"parseErrors"` } func extJSONRepr(s string) string { - var value struct { - D struct { - Repr string `json:"$numberDecimal"` - } `json:"d"` - } - err := json.Unmarshal([]byte(s), &value) - if err != nil { - panic(err) - } - return value.D.Repr + var value struct { + D struct { + Repr string `json:"$numberDecimal"` + } `json:"d"` + } + err := json.Unmarshal([]byte(s), &value) + if err != nil { + panic(err) + } + return value.D.Repr } func (s *S) TestDecimalTests(c *C) { - // These also conform to the spec and are used by Go elsewhere. - // (e.g. math/big won't parse "Infinity"). - goStr := func(s string) string { - switch s { - case "Infinity": - return "Inf" - case "-Infinity": - return "-Inf" - } - return s - } + // These also conform to the spec and are used by Go elsewhere. + // (e.g. math/big won't parse "Infinity"). + goStr := func(s string) string { + switch s { + case "Infinity": + return "Inf" + case "-Infinity": + return "-Inf" + } + return s + } - for _, testEntry := range decimalTestsJSON { - testFile := testEntry.file + for _, testEntry := range decimalTestsJSON { + testFile := testEntry.file - var tests decimalTests - err := json.Unmarshal([]byte(testEntry.json), &tests) - c.Assert(err, IsNil) + var tests decimalTests + err := json.Unmarshal([]byte(testEntry.json), &tests) + c.Assert(err, IsNil) - for _, test := range tests.Valid { - c.Logf("Running %s test: %s", testFile, test.Description) + for _, test := range tests.Valid { + c.Logf("Running %s test: %s", testFile, test.Description) - test.BSON = strings.ToLower(test.BSON) + test.BSON = strings.ToLower(test.BSON) - // Unmarshal value from BSON data. - bsonData, err := hex.DecodeString(test.BSON) - var bsonValue struct{ D interface{} } - err = bson.Unmarshal(bsonData, &bsonValue) - c.Assert(err, IsNil) - dec128, ok := bsonValue.D.(bson.Decimal128) - c.Assert(ok, Equals, true) + // Unmarshal value from BSON data. + bsonData, err := hex.DecodeString(test.BSON) + var bsonValue struct{ D interface{} } + err = bson.Unmarshal(bsonData, &bsonValue) + c.Assert(err, IsNil) + dec128, ok := bsonValue.D.(bson.Decimal128) + c.Assert(ok, Equals, true) - // Extract ExtJSON representations (canonical and not). - extjRepr := extJSONRepr(test.ExtJSON) - cextjRepr := extjRepr - if test.CanonicalExtJSON != "" { - cextjRepr = extJSONRepr(test.CanonicalExtJSON) - } + // Extract ExtJSON representations (canonical and not). + extjRepr := extJSONRepr(test.ExtJSON) + cextjRepr := extjRepr + if test.CanonicalExtJSON != "" { + cextjRepr = extJSONRepr(test.CanonicalExtJSON) + } - wantRepr := goStr(cextjRepr) + wantRepr := goStr(cextjRepr) - // Generate canonical representation. - c.Assert(dec128.String(), Equals, wantRepr) + // Generate canonical representation. + c.Assert(dec128.String(), Equals, wantRepr) - // Parse original canonical representation. - parsed, err := bson.ParseDecimal128(cextjRepr) - c.Assert(err, IsNil) - c.Assert(parsed.String(), Equals, wantRepr) + // Parse original canonical representation. + parsed, err := bson.ParseDecimal128(cextjRepr) + c.Assert(err, IsNil) + c.Assert(parsed.String(), Equals, wantRepr) - // Parse non-canonical representation. - parsed, err = bson.ParseDecimal128(extjRepr) - c.Assert(err, IsNil) - c.Assert(parsed.String(), Equals, wantRepr) + // Parse non-canonical representation. + parsed, err = bson.ParseDecimal128(extjRepr) + c.Assert(err, IsNil) + c.Assert(parsed.String(), Equals, wantRepr) - // Parse Go canonical representation (Inf vs. Infinity). - parsed, err = bson.ParseDecimal128(wantRepr) - c.Assert(err, IsNil) - c.Assert(parsed.String(), Equals, wantRepr) + // Parse Go canonical representation (Inf vs. Infinity). + parsed, err = bson.ParseDecimal128(wantRepr) + c.Assert(err, IsNil) + c.Assert(parsed.String(), Equals, wantRepr) - // Marshal original value back into BSON data. - data, err := bson.Marshal(bsonValue) - c.Assert(err, IsNil) - c.Assert(hex.EncodeToString(data), Equals, test.BSON) + // Marshal original value back into BSON data. + data, err := bson.Marshal(bsonValue) + c.Assert(err, IsNil) + c.Assert(hex.EncodeToString(data), Equals, test.BSON) - if test.Lossy { - continue - } + if test.Lossy { + continue + } - // Marshal the parsed canonical representation. - var parsedValue struct{ D interface{} } - parsedValue.D = parsed - data, err = bson.Marshal(parsedValue) - c.Assert(err, IsNil) - c.Assert(hex.EncodeToString(data), Equals, test.BSON) - } + // Marshal the parsed canonical representation. + var parsedValue struct{ D interface{} } + parsedValue.D = parsed + data, err = bson.Marshal(parsedValue) + c.Assert(err, IsNil) + c.Assert(hex.EncodeToString(data), Equals, test.BSON) + } - for _, test := range tests.ParseErrors { - c.Logf("Running %s parse error test: %s (string %q)", testFile, test.Description, test.String) + for _, test := range tests.ParseErrors { + c.Logf("Running %s parse error test: %s (string %q)", testFile, test.Description, test.String) - _, err := bson.ParseDecimal128(test.String) - quoted := regexp.QuoteMeta(fmt.Sprintf("%q", test.String)) - c.Assert(err, ErrorMatches, `cannot parse `+quoted+` as a decimal128`) - } - } + _, err := bson.ParseDecimal128(test.String) + quoted := regexp.QuoteMeta(fmt.Sprintf("%q", test.String)) + c.Assert(err, ErrorMatches, `cannot parse `+quoted+` as a decimal128`) + } + } } const decBenchNum = "9.999999999999999999999999999999999E+6144" func (s *S) BenchmarkDecimal128String(c *C) { - d, err := bson.ParseDecimal128(decBenchNum) - c.Assert(err, IsNil) - c.Assert(d.String(), Equals, decBenchNum) + d, err := bson.ParseDecimal128(decBenchNum) + c.Assert(err, IsNil) + c.Assert(d.String(), Equals, decBenchNum) - c.ResetTimer() - for i := 0; i < c.N; i++ { - d.String() - } + c.ResetTimer() + for i := 0; i < c.N; i++ { + _ = d.String() + } } func (s *S) BenchmarkDecimal128Parse(c *C) { - var err error - c.ResetTimer() - for i := 0; i < c.N; i++ { - _, err = bson.ParseDecimal128(decBenchNum) - } - if err != nil { - panic(err) - } + var err error + c.ResetTimer() + for i := 0; i < c.N; i++ { + _, err = bson.ParseDecimal128(decBenchNum) + } + if err != nil { + panic(err) + } } var decimalTestsJSON = []struct{ file, json string }{ - {"decimal128-1.json", ` + {"decimal128-1.json", ` { "description": "Decimal128", "bson_type": "0x13", @@ -502,7 +502,7 @@ var decimalTestsJSON = []struct{ file, json string }{ } `}, - {"decimal128-2.json", ` + {"decimal128-2.json", ` { "description": "Decimal128", "bson_type": "0x13", @@ -1297,7 +1297,7 @@ var decimalTestsJSON = []struct{ file, json string }{ } `}, - {"decimal128-3.json", ` + {"decimal128-3.json", ` { "description": "Decimal128", "bson_type": "0x13", @@ -3071,7 +3071,7 @@ var decimalTestsJSON = []struct{ file, json string }{ } `}, - {"decimal128-4.json", ` + {"decimal128-4.json", ` { "description": "Decimal128", "bson_type": "0x13", @@ -3239,7 +3239,7 @@ var decimalTestsJSON = []struct{ file, json string }{ } `}, - {"decimal128-5.json", ` + {"decimal128-5.json", ` { "description": "Decimal128", "bson_type": "0x13", @@ -3643,7 +3643,7 @@ var decimalTestsJSON = []struct{ file, json string }{ } `}, - {"decimal128-6.json", ` + {"decimal128-6.json", ` { "description": "Decimal128", "bson_type": "0x13", @@ -3777,7 +3777,7 @@ var decimalTestsJSON = []struct{ file, json string }{ } `}, - {"decimal128-7.json", ` + {"decimal128-7.json", ` { "description": "Decimal128", "bson_type": "0x13", diff --git a/bson/decode.go b/bson/decode.go index 417a4d14b..3b9e2856d 100644 --- a/bson/decode.go +++ b/bson/decode.go @@ -457,7 +457,7 @@ func (d *decoder) dropElem(kind byte) { } d.i += l case 0x06: // undefined - case 0x07: // objectID + case 0x07: // objectId d.i += 12 case 0x08: k := d.readByte() @@ -629,7 +629,7 @@ func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { if setter := getSetter(outt, out); setter != nil { err := setter.SetBSON(Raw{kind, d.in[start:d.i]}) - if err == SetZero { + if err == ErrSetZero { out.Set(reflect.Zero(outt)) return true } diff --git a/bson/encode.go b/bson/encode.go index 2ce66339f..75e503b57 100644 --- a/bson/encode.go +++ b/bson/encode.go @@ -331,7 +331,7 @@ func (e *encoder) addElem(name string, v reflect.Value, minSize bool) { // Stored as int64 e.addElemName(0x12, name) - e.addInt64(int64(v.Int()/1e6)) + e.addInt64(int64(v.Int() / 1e6)) default: i := v.Int() if (minSize || v.Type().Kind() != reflect.Int64) && i >= math.MinInt32 && i <= math.MaxInt32 { diff --git a/bson/json_test.go b/bson/json_test.go index 880fb87c2..fb8a4ac5b 100644 --- a/bson/json_test.go +++ b/bson/json_test.go @@ -65,7 +65,7 @@ var jsonTests = []jsonTest{ // $regex { - a: bson.RegEx{"pattern", "options"}, + a: bson.RegEx{Pattern: "pattern", Options: "options"}, b: `{"$regex":"pattern","$options":"options"}`, }, diff --git a/bulk_test.go b/bulk_test.go index b1a5fbb3c..fa91dc44c 100644 --- a/bulk_test.go +++ b/bulk_test.go @@ -64,7 +64,7 @@ func (s *S) TestBulkInsertError(c *C) { c.Assert(mgo.IsDup(err), Equals, true) type doc struct { - N int `_id` + N int `bson:"_id"` } var res []doc err = coll.Find(nil).Sort("_id").All(&res) @@ -85,7 +85,7 @@ func (s *S) TestBulkInsertErrorUnordered(c *C) { c.Assert(err, ErrorMatches, ".*duplicate key.*") type doc struct { - N int `_id` + N int `bson:"_id"` } var res []doc err = coll.Find(nil).Sort("_id").All(&res) @@ -110,7 +110,7 @@ func (s *S) TestBulkInsertErrorUnorderedSplitBatch(c *C) { const total = 4096 type doc struct { - Id int `_id` + Id int `bson:"_id"` } docs := make([]interface{}, total) for i := 0; i < total; i++ { diff --git a/cluster.go b/cluster.go index 81e4f7ff5..7fc639c24 100644 --- a/cluster.go +++ b/cluster.go @@ -157,7 +157,7 @@ func (cluster *mongoCluster) isMaster(socket *mongoSocket, result *isMasterResul if cluster.appName != "" { metaInfo["application"] = bson.M{"name": cluster.appName} } - err := session.Run(bson.D{{"isMaster", 1}, {"client", metaInfo}}, result) + err := session.Run(bson.D{{Name: "isMaster", Value: 1}, {Name: "client", Value: metaInfo}}, result) session.Close() return err } @@ -667,7 +667,6 @@ func (cluster *mongoCluster) AcquireSocket(mode Mode, slaveOk bool, syncTimeout } return s, nil } - panic("unreached") } func (cluster *mongoCluster) CacheIndex(cacheKey string, exists bool) { diff --git a/cluster_test.go b/cluster_test.go index 1436cc317..539422be7 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -500,7 +500,7 @@ func (s *S) TestModePrimaryHiccup(c *C) { sessions[i].Close() } - // Kill the master, but bring it back immediatelly. + // Kill the master, but bring it back immediately. host := result.Host s.Stop(host) s.StartAll() @@ -1526,7 +1526,7 @@ func (s *S) TestRemovalOfClusterMember(c *C) { "40023": `{_id: 3, host: "127.0.0.1:40023", priority: 0, tags: {rs2: "c"}}`, } master.Refresh() - master.Run(bson.D{{"$eval", `rs.add(` + config[hostPort(slaveAddr)] + `)`}}, nil) + master.Run(bson.D{{Name: "$eval", Value: `rs.add(` + config[hostPort(slaveAddr)] + `)`}}, nil) master.Close() slave.Close() @@ -1541,7 +1541,7 @@ func (s *S) TestRemovalOfClusterMember(c *C) { c.Logf("========== Removing slave: %s ==========", slaveAddr) - master.Run(bson.D{{"$eval", `rs.remove("` + slaveAddr + `")`}}, nil) + master.Run(bson.D{{Name: "$eval", Value: `rs.remove("` + slaveAddr + `")`}}, nil) master.Refresh() @@ -1563,7 +1563,7 @@ func (s *S) TestRemovalOfClusterMember(c *C) { } live := master.LiveServers() if len(live) != 2 { - c.Errorf("Removed server still considered live: %#s", live) + c.Errorf("Removed server still considered live: %v", live) } c.Log("========== Test succeeded. ==========") @@ -1812,6 +1812,7 @@ func (s *S) TestPrimaryShutdownOnAuthShard(c *C) { c.Assert(err, IsNil) count, err := coll.Count() + c.Assert(err, IsNil) c.Assert(count > 1, Equals, true) } @@ -1977,13 +1978,13 @@ func (s *S) TestSelectServers(c *C) { var result struct{ Host string } session.Refresh() - session.SelectServers(bson.D{{"rs1", "b"}}) + session.SelectServers(bson.D{{Name: "rs1", Value: "b"}}) err = session.Run("serverStatus", &result) c.Assert(err, IsNil) c.Assert(hostPort(result.Host), Equals, "40012") session.Refresh() - session.SelectServers(bson.D{{"rs1", "c"}}) + session.SelectServers(bson.D{{Name: "rs1", Value: "c"}}) err = session.Run("serverStatus", &result) c.Assert(err, IsNil) c.Assert(hostPort(result.Host), Equals, "40013") @@ -2035,7 +2036,7 @@ func (s *S) TestSelectServersWithMongos(c *C) { mongos.SetMode(mgo.Monotonic, true) mongos.Refresh() - mongos.SelectServers(bson.D{{"rs2", slave1}}) + mongos.SelectServers(bson.D{{Name: "rs2", Value: slave1}}) coll := mongos.DB("mydb").C("mycoll") result := &struct{}{} for i := 0; i != 5; i++ { @@ -2044,7 +2045,7 @@ func (s *S) TestSelectServersWithMongos(c *C) { } mongos.Refresh() - mongos.SelectServers(bson.D{{"rs2", slave2}}) + mongos.SelectServers(bson.D{{Name: "rs2", Value: slave2}}) coll = mongos.DB("mydb").C("mycoll") for i := 0; i != 7; i++ { err := coll.Find(nil).One(result) diff --git a/dbtest/dbserver.go b/dbtest/dbserver.go index 4fe530c90..b74280801 100644 --- a/dbtest/dbserver.go +++ b/dbtest/dbserver.go @@ -142,7 +142,7 @@ func (dbs *DBServer) Session() *mgo.Session { // checkSessions ensures all mgo sessions opened were properly closed. // For slightly faster tests, it may be disabled setting the -// environmnet variable CHECK_SESSIONS to 0. +// environment variable CHECK_SESSIONS to 0. func (dbs *DBServer) checkSessions() { if check := os.Getenv("CHECK_SESSIONS"); check == "0" || dbs.server == nil || dbs.session == nil { return diff --git a/dbtest/dbserver_test.go b/dbtest/dbserver_test.go index f0576c25c..b3cc45a8a 100644 --- a/dbtest/dbserver_test.go +++ b/dbtest/dbserver_test.go @@ -78,6 +78,8 @@ func (s *S) TestStop(c *C) { // Server should not be running anymore. session, err = mgo.DialWithTimeout(addr, 500*time.Millisecond) + c.Assert(err, IsNil) + if session != nil { session.Close() c.Fatalf("Stop did not stop the server") diff --git a/gridfs.go b/gridfs.go index 71ca609fa..0954b166b 100644 --- a/gridfs.go +++ b/gridfs.go @@ -39,6 +39,26 @@ import ( "github.com/globalsign/mgo/bson" ) +// GridFS stores files in two collections: +// +// - chunks stores the binary chunks. For details, see the chunks Collection. +// - files stores the file’s metadata. For details, see the files Collection. +// +// GridFS places the collections in a common bucket by prefixing each with the bucket name. +// By default, GridFS uses two collections with a bucket named fs: +// +// - fs.files +// - fs.chunks +// +// You can choose a different bucket name, as well as create multiple buckets in a single database. +// The full collection name, which includes the bucket name, is subject to the namespace length limit. +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/core/gridfs/ +// https://docs.mongodb.com/manual/core/gridfs/#gridfs-chunks-collection +// https://docs.mongodb.com/manual/core/gridfs/#gridfs-files-collection +// type GridFS struct { Files *Collection Chunks *Collection @@ -52,6 +72,7 @@ const ( gfsWriting gfsFileMode = 2 ) +// GridFile document in files collection type GridFile struct { m sync.Mutex c sync.Cond @@ -73,19 +94,19 @@ type GridFile struct { } type gfsFile struct { - Id interface{} "_id" - ChunkSize int "chunkSize" - UploadDate time.Time "uploadDate" - Length int64 ",minsize" + Id interface{} `bson:"_id"` + ChunkSize int `bson:"chunkSize"` + UploadDate time.Time `bson:"uploadDate"` + Length int64 `bson:",minsize"` MD5 string - Filename string ",omitempty" - ContentType string "contentType,omitempty" - Metadata *bson.Raw ",omitempty" + Filename string `bson:",omitempty"` + ContentType string `bson:"contentType,omitempty"` + Metadata *bson.Raw `bson:",omitempty"` } type gfsChunk struct { - Id interface{} "_id" - FilesId interface{} "files_id" + Id interface{} `bson:"_id"` + FilesId interface{} `bson:"files_id"` N int Data []byte } @@ -319,12 +340,12 @@ func (gfs *GridFS) RemoveId(id interface{}) error { if err != nil { return err } - _, err = gfs.Chunks.RemoveAll(bson.D{{"files_id", id}}) + _, err = gfs.Chunks.RemoveAll(bson.D{{Name: "files_id", Value: id}}) return err } type gfsDocId struct { - Id interface{} "_id" + Id interface{} `bson:"_id"` } // Remove deletes all files with the provided name from the GridFS. @@ -411,7 +432,7 @@ func (file *GridFile) ContentType() string { return file.doc.ContentType } -// ContentType changes the optional file content type. An empty string may be +// SetContentType changes the optional file content type. An empty string may be // used to unset it. // // It is a runtime error to call this function when the file is not open @@ -530,7 +551,7 @@ func (file *GridFile) completeWrite() { file.err = file.gfs.Files.Insert(file.doc) } if file.err != nil { - file.gfs.Chunks.RemoveAll(bson.D{{"files_id", file.doc.Id}}) + file.gfs.Chunks.RemoveAll(bson.D{{Name: "files_id", Value: file.doc.Id}}) } if file.err == nil { index := Index{ @@ -734,7 +755,7 @@ func (file *GridFile) getChunk() (data []byte, err error) { } else { debugf("GridFile %p: Fetching chunk %d", file, file.chunk) var doc gfsChunk - err = file.gfs.Chunks.Find(bson.D{{"files_id", file.doc.Id}, {"n", file.chunk}}).One(&doc) + err = file.gfs.Chunks.Find(bson.D{{Name: "files_id", Value: file.doc.Id}, {Name: "n", Value: file.chunk}}).One(&doc) data = doc.Data } file.chunk++ @@ -750,7 +771,7 @@ func (file *GridFile) getChunk() (data []byte, err error) { defer session.Close() chunks = chunks.With(session) var doc gfsChunk - cache.err = chunks.Find(bson.D{{"files_id", id}, {"n", n}}).One(&doc) + cache.err = chunks.Find(bson.D{{Name: "files_id", Value: id}, {Name: "n", Value: n}}).One(&doc) cache.data = doc.Data cache.wait.Unlock() }(file.doc.Id, file.chunk) diff --git a/internal/json/decode.go b/internal/json/decode.go index 2171d91a7..d5ca1f9a8 100644 --- a/internal/json/decode.go +++ b/internal/json/decode.go @@ -899,7 +899,7 @@ func (d *decodeState) name(v reflect.Value) { } // Check for unmarshaler on func field itself. - u, ut, pv = d.indirect(v, false) + u, _, _ = d.indirect(v, false) if u != nil { d.off = nameStart err := u.UnmarshalJSON(d.next()) diff --git a/internal/json/decode_test.go b/internal/json/decode_test.go index 30e46ca44..e921d48bd 100644 --- a/internal/json/decode_test.go +++ b/internal/json/decode_test.go @@ -109,7 +109,7 @@ var ( umstructXY = ustructText{unmarshalerText{"x", "y"}} ummapType = map[unmarshalerText]bool{} - ummapXY = map[unmarshalerText]bool{unmarshalerText{"x", "y"}: true} + ummapXY = map[unmarshalerText]bool{{"x", "y"}: true} ) // Test data structures for anonymous fields. diff --git a/internal/json/encode.go b/internal/json/encode.go index 67a0f0062..e4b8f8648 100644 --- a/internal/json/encode.go +++ b/internal/json/encode.go @@ -209,6 +209,8 @@ func (e *UnsupportedTypeError) Error() string { return "json: unsupported type: " + e.Type.String() } +// An UnsupportedValueError is returned by Marshal when attempting +// to encode an unsupported value. type UnsupportedValueError struct { Value reflect.Value Str string @@ -218,7 +220,7 @@ func (e *UnsupportedValueError) Error() string { return "json: unsupported value: " + e.Str } -// Before Go 1.2, an InvalidUTF8Error was returned by Marshal when +// InvalidUTF8Error before Go 1.2, an InvalidUTF8Error was returned by Marshal when // attempting to encode a string value with invalid UTF-8 sequences. // As of Go 1.2, Marshal instead coerces the string to valid UTF-8 by // replacing invalid bytes with the Unicode replacement rune U+FFFD. @@ -232,6 +234,8 @@ func (e *InvalidUTF8Error) Error() string { return "json: invalid UTF-8 in string: " + strconv.Quote(e.S) } +// A MarshalerError is returned by Marshal when attempting +// to marshal an invalid JSON type MarshalerError struct { Type reflect.Type Err error diff --git a/internal/json/stream_test.go b/internal/json/stream_test.go index 4ebeaba96..6c40f0ab2 100644 --- a/internal/json/stream_test.go +++ b/internal/json/stream_test.go @@ -282,7 +282,7 @@ type decodeThis struct { v interface{} } -var tokenStreamCases []tokenStreamCase = []tokenStreamCase{ +var tokenStreamCases = []tokenStreamCase{ // streaming token cases {json: `10`, expTokens: []interface{}{float64(10)}}, {json: ` [10] `, expTokens: []interface{}{ diff --git a/internal/sasl/sasl.go b/internal/sasl/sasl.go index 870a0add4..25a537426 100644 --- a/internal/sasl/sasl.go +++ b/internal/sasl/sasl.go @@ -26,7 +26,8 @@ import ( "unsafe" ) -type saslStepper interface { +// Stepper interface for saslSession +type Stepper interface { Step(serverData []byte) (clientData []byte, done bool, err error) Close() } @@ -50,7 +51,8 @@ func initSASL() { } } -func New(username, password, mechanism, service, host string) (saslStepper, error) { +// New creates a new saslSession +func New(username, password, mechanism, service, host string) (Stepper, error) { initOnce.Do(initSASL) if initError != nil { return nil, initError diff --git a/internal/scram/scram.go b/internal/scram/scram.go index 80cda9135..d3ddd02fd 100644 --- a/internal/scram/scram.go +++ b/internal/scram/scram.go @@ -24,7 +24,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// Pacakage scram implements a SCRAM-{SHA-1,etc} client per RFC5802. +// Package scram implements a SCRAM-{SHA-1,etc} client per RFC5802. // // http://tools.ietf.org/html/rfc5802 // @@ -96,7 +96,7 @@ func (c *Client) Out() []byte { return c.out.Bytes() } -// Err returns the error that ocurred, or nil if there were no errors. +// Err returns the error that occurred, or nil if there were no errors. func (c *Client) Err() error { return c.err } @@ -133,7 +133,7 @@ func (c *Client) Step(in []byte) bool { func (c *Client) step1(in []byte) error { if len(c.clientNonce) == 0 { const nonceLen = 6 - buf := make([]byte, nonceLen + b64.EncodedLen(nonceLen)) + buf := make([]byte, nonceLen+b64.EncodedLen(nonceLen)) if _, err := rand.Read(buf[:nonceLen]); err != nil { return fmt.Errorf("cannot read random SCRAM-SHA-1 nonce from operating system: %v", err) } diff --git a/log.go b/log.go index 53eb4237b..d83779498 100644 --- a/log.go +++ b/log.go @@ -34,15 +34,15 @@ import ( // --------------------------------------------------------------------------- // Logging integration. -// Avoid importing the log type information unnecessarily. There's a small cost +// LogLogger avoid importing the log type information unnecessarily. There's a small cost // associated with using an interface rather than the type. Depending on how // often the logger is plugged in, it would be worth using the type instead. -type log_Logger interface { +type logLogger interface { Output(calldepth int, s string) error } var ( - globalLogger log_Logger + globalLogger logLogger globalDebug bool globalMutex sync.Mutex ) @@ -53,8 +53,8 @@ var ( // the application starts. Having raceDetector as a constant, the compiler // should elide the locks altogether in actual use. -// Specify the *log.Logger object where log messages should be sent to. -func SetLogger(logger log_Logger) { +// SetLogger specify the *log.Logger object where log messages should be sent to. +func SetLogger(logger logLogger) { if raceDetector { globalMutex.Lock() defer globalMutex.Unlock() @@ -62,7 +62,7 @@ func SetLogger(logger log_Logger) { globalLogger = logger } -// Enable the delivery of debug messages to the logger. Only meaningful +// SetDebug enable the delivery of debug messages to the logger. Only meaningful // if a logger is also set. func SetDebug(debug bool) { if raceDetector { diff --git a/server.go b/server.go index 7b31e243f..7ad955255 100644 --- a/server.go +++ b/server.go @@ -143,7 +143,6 @@ func (server *mongoServer) AcquireSocket(poolLimit int, timeout time.Duration) ( } return } - panic("unreachable") } // Connect establishes a new connection to the server. This should @@ -306,7 +305,7 @@ func (server *mongoServer) pinger(loop bool) { } op := queryOp{ collection: "admin.$cmd", - query: bson.D{{"ping", 1}}, + query: bson.D{{Name: "ping", Value: 1}}, flags: flagSlaveOk, limit: -1, } diff --git a/session.go b/session.go index 2b383ad40..074f48688 100644 --- a/session.go +++ b/session.go @@ -44,23 +44,32 @@ import ( "github.com/globalsign/mgo/bson" ) +// Mode read preference mode. See Eventual, Monotonic and Strong for details +// +// Relevant documentation on read preference modes: +// +// http://docs.mongodb.org/manual/reference/read-preference/ +// type Mode int const ( - // Relevant documentation on read preference modes: - // - // http://docs.mongodb.org/manual/reference/read-preference/ - // - Primary Mode = 2 // Default mode. All operations read from the current replica set primary. - PrimaryPreferred Mode = 3 // Read from the primary if available. Read from the secondary otherwise. - Secondary Mode = 4 // Read from one of the nearest secondary members of the replica set. - SecondaryPreferred Mode = 5 // Read from one of the nearest secondaries if available. Read from primary otherwise. - Nearest Mode = 6 // Read from one of the nearest members, irrespective of it being primary or secondary. - - // Read preference modes are specific to mgo: - Eventual Mode = 0 // Same as Nearest, but may change servers between reads. - Monotonic Mode = 1 // Same as SecondaryPreferred before first write. Same as Primary after first write. - Strong Mode = 2 // Same as Primary. + // Primary mode is default mode. All operations read from the current replica set primary. + Primary Mode = 2 + // PrimaryPreferred mode: read from the primary if available. Read from the secondary otherwise. + PrimaryPreferred Mode = 3 + // Secondary mode: read from one of the nearest secondary members of the replica set. + Secondary Mode = 4 + // SecondaryPreferred mode: read from one of the nearest secondaries if available. Read from primary otherwise. + SecondaryPreferred Mode = 5 + // Nearest mode: read from one of the nearest members, irrespective of it being primary or secondary. + Nearest Mode = 6 + + // Eventual mode is specific to mgo, and is same as Nearest, but may change servers between reads. + Eventual Mode = 0 + // Monotonic mode is specifc to mgo, and is same as SecondaryPreferred before first write. Same as Primary after first write. + Monotonic Mode = 1 + // Strong mode is specific to mgo, and is same as Primary. + Strong Mode = 2 ) // mgo.v3: Drop Strong mode, suffix all modes with "Mode". @@ -84,7 +93,7 @@ type Session struct { creds []Credential dialCred *Credential safeOp *queryOp - cluster_ *mongoCluster + mgoCluster *mongoCluster slaveSocket *mongoSocket masterSocket *mongoSocket m sync.RWMutex @@ -93,17 +102,30 @@ type Session struct { slaveOk bool } +// Database holds collections of documents +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/core/databases-and-collections/#databases +// type Database struct { Session *Session Name string } +// Collection stores documents +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/core/databases-and-collections/#collections +// type Collection struct { Database *Database Name string // "collection" FullName string // "db.collection" } +// Query keeps info on the query. type Query struct { m sync.Mutex session *Session @@ -117,13 +139,19 @@ type query struct { } type getLastError struct { - CmdName int "getLastError,omitempty" - W interface{} "w,omitempty" - WTimeout int "wtimeout,omitempty" - FSync bool "fsync,omitempty" - J bool "j,omitempty" + CmdName int `bson:"getLastError,omitempty"` + W interface{} `bson:"w,omitempty"` + WTimeout int `bson:"wtimeout,omitempty"` + FSync bool `bson:"fsync,omitempty"` + J bool `bson:"j,omitempty"` } +// Iter stores informations about a Cursor +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/tutorial/iterate-a-cursor/ +// type Iter struct { m sync.Mutex gotReply sync.Cond @@ -142,8 +170,11 @@ type Iter struct { } var ( + // ErrNotFound error returned when a document could not be found ErrNotFound = errors.New("not found") - ErrCursor = errors.New("invalid cursor") + // ErrCursor error returned when trying to retrieve documents from + // an invalid cursor + ErrCursor = errors.New("invalid cursor") ) const ( @@ -599,7 +630,7 @@ func extractURL(s string) (*urlInfo, error) { func newSession(consistency Mode, cluster *mongoCluster, timeout time.Duration) (session *Session) { cluster.Acquire() session = &Session{ - cluster_: cluster, + mgoCluster: cluster, syncTimeout: timeout, sockTimeout: timeout, poolLimit: 4096, @@ -627,9 +658,24 @@ func copySession(session *Session, keepCreds bool) (s *Session) { } else if session.dialCred != nil { creds = []Credential{*session.dialCred} } - scopy := *session - scopy.m = sync.RWMutex{} - scopy.creds = creds + scopy := Session{ + defaultdb: session.defaultdb, + sourcedb: session.sourcedb, + syncTimeout: session.syncTimeout, + sockTimeout: session.sockTimeout, + poolLimit: session.poolLimit, + consistency: session.consistency, + creds: creds, + dialCred: session.dialCred, + safeOp: session.safeOp, + mgoCluster: session.mgoCluster, + slaveSocket: session.slaveSocket, + masterSocket: session.masterSocket, + m: sync.RWMutex{}, + queryConfig: session.queryConfig, + bypassValidation: session.bypassValidation, + slaveOk: session.slaveOk, + } s = &scopy debugf("New session %p on cluster %p (copy from %p)", s, cluster, session) return s @@ -683,9 +729,9 @@ func (db *Database) C(name string) *Collection { // https://docs.mongodb.com/manual/reference/method/db.createView/ // func (db *Database) CreateView(view string, source string, pipeline interface{}, collation *Collation) error { - command := bson.D{{"create", view}, {"viewOn", source}, {"pipeline", pipeline}} + command := bson.D{{Name: "create", Value: view}, {Name: "viewOn", Value: source}, {Name: "pipeline", Value: pipeline}} if collation != nil { - command = append(command, bson.DocElem{"collation", collation}) + command = append(command, bson.DocElem{Name: "collation", Value: collation}) } return db.Run(command, nil) } @@ -910,23 +956,51 @@ type User struct { UserSource string `bson:"userSource,omitempty"` } +// Role available role for users +// +// Relevant documentation: +// +// http://docs.mongodb.org/manual/reference/user-privileges/ +// type Role string const ( - // Relevant documentation: - // - // http://docs.mongodb.org/manual/reference/user-privileges/ - // - RoleRoot Role = "root" - RoleRead Role = "read" - RoleReadAny Role = "readAnyDatabase" - RoleReadWrite Role = "readWrite" + // RoleRoot provides access to the operations and all the resources + // of the readWriteAnyDatabase, dbAdminAnyDatabase, userAdminAnyDatabase, + // clusterAdmin roles, restore, and backup roles combined. + RoleRoot Role = "root" + // RoleRead provides the ability to read data on all non-system collections + // and on the following system collections: system.indexes, system.js, and + // system.namespaces collections on a specific database. + RoleRead Role = "read" + // RoleReadAny provides the same read-only permissions as read, except it + // applies to it applies to all but the local and config databases in the cluster. + // The role also provides the listDatabases action on the cluster as a whole. + RoleReadAny Role = "readAnyDatabase" + //RoleReadWrite provides all the privileges of the read role plus ability to modify data on + //all non-system collections and the system.js collection on a specific database. + RoleReadWrite Role = "readWrite" + // RoleReadWriteAny provides the same read and write permissions as readWrite, except it + // applies to all but the local and config databases in the cluster. The role also provides + // the listDatabases action on the cluster as a whole. RoleReadWriteAny Role = "readWriteAnyDatabase" - RoleDBAdmin Role = "dbAdmin" - RoleDBAdminAny Role = "dbAdminAnyDatabase" - RoleUserAdmin Role = "userAdmin" + // RoleDBAdmin provides all the privileges of the dbAdmin role on a specific database + RoleDBAdmin Role = "dbAdmin" + // RoleDBAdminAny provides all the privileges of the dbAdmin role on all databases + RoleDBAdminAny Role = "dbAdminAnyDatabase" + // RoleUserAdmin Provides the ability to create and modify roles and users on the + // current database. This role also indirectly provides superuser access to either + // the database or, if scoped to the admin database, the cluster. The userAdmin role + // allows users to grant any user any privilege, including themselves. + RoleUserAdmin Role = "userAdmin" + // RoleUserAdminAny provides the same access to user administration operations as userAdmin, + // except it applies to all but the local and config databases in the cluster RoleUserAdminAny Role = "userAdminAnyDatabase" + // RoleClusterAdmin Provides the greatest cluster-management access. This role combines + // the privileges granted by the clusterManager, clusterMonitor, and hostManager roles. + // Additionally, the role provides the dropDatabase action. RoleClusterAdmin Role = "clusterAdmin" + // TODO some roles are missing: dbOwner/clusterManager/clusterMonitor/hostManager/backup/restore ) // UpsertUser updates the authentication credentials and the roles for @@ -972,32 +1046,32 @@ func (db *Database) UpsertUser(user *User) error { if user.Password != "" { psum := md5.New() psum.Write([]byte(user.Username + ":mongo:" + user.Password)) - set = append(set, bson.DocElem{"pwd", hex.EncodeToString(psum.Sum(nil))}) - unset = append(unset, bson.DocElem{"userSource", 1}) + set = append(set, bson.DocElem{Name: "pwd", Value: hex.EncodeToString(psum.Sum(nil))}) + unset = append(unset, bson.DocElem{Name: "userSource", Value: 1}) } else if user.PasswordHash != "" { - set = append(set, bson.DocElem{"pwd", user.PasswordHash}) - unset = append(unset, bson.DocElem{"userSource", 1}) + set = append(set, bson.DocElem{Name: "pwd", Value: user.PasswordHash}) + unset = append(unset, bson.DocElem{Name: "userSource", Value: 1}) } if user.UserSource != "" { - set = append(set, bson.DocElem{"userSource", user.UserSource}) - unset = append(unset, bson.DocElem{"pwd", 1}) + set = append(set, bson.DocElem{Name: "userSource", Value: user.UserSource}) + unset = append(unset, bson.DocElem{Name: "pwd", Value: 1}) } if user.Roles != nil || user.OtherDBRoles != nil { - set = append(set, bson.DocElem{"roles", user.Roles}) + set = append(set, bson.DocElem{Name: "roles", Value: user.Roles}) if len(user.OtherDBRoles) > 0 { - set = append(set, bson.DocElem{"otherDBRoles", user.OtherDBRoles}) + set = append(set, bson.DocElem{Name: "otherDBRoles", Value: user.OtherDBRoles}) } else { - unset = append(unset, bson.DocElem{"otherDBRoles", 1}) + unset = append(unset, bson.DocElem{Name: "otherDBRoles", Value: 1}) } } users := db.C("system.users") - err = users.Update(bson.D{{"user", user.Username}}, bson.D{{"$unset", unset}, {"$set", set}}) + err = users.Update(bson.D{{Name: "user", Value: user.Username}}, bson.D{{Name: "$unset", Value: unset}, {Name: "$set", Value: set}}) if err == ErrNotFound { - set = append(set, bson.DocElem{"user", user.Username}) + set = append(set, bson.DocElem{Name: "user", Value: user.Username}) if user.Roles == nil && user.OtherDBRoles == nil { // Roles must be sent, as it's the way MongoDB distinguishes // old-style documents from new-style documents in pre-2.6. - set = append(set, bson.DocElem{"roles", user.Roles}) + set = append(set, bson.DocElem{Name: "roles", Value: user.Roles}) } err = users.Insert(set) } @@ -1021,9 +1095,9 @@ func isAuthError(err error) bool { func (db *Database) runUserCmd(cmdName string, user *User) error { cmd := make(bson.D, 0, 16) - cmd = append(cmd, bson.DocElem{cmdName, user.Username}) + cmd = append(cmd, bson.DocElem{Name: cmdName, Value: user.Username}) if user.Password != "" { - cmd = append(cmd, bson.DocElem{"pwd", user.Password}) + cmd = append(cmd, bson.DocElem{Name: "pwd", Value: user.Password}) } var roles []interface{} for _, role := range user.Roles { @@ -1031,11 +1105,11 @@ func (db *Database) runUserCmd(cmdName string, user *User) error { } for db, dbroles := range user.OtherDBRoles { for _, role := range dbroles { - roles = append(roles, bson.D{{"role", role}, {"db", db}}) + roles = append(roles, bson.D{{Name: "role", Value: role}, {Name: "db", Value: db}}) } } if roles != nil || user.Roles != nil || cmdName == "createUser" { - cmd = append(cmd, bson.DocElem{"roles", roles}) + cmd = append(cmd, bson.DocElem{Name: "roles", Value: roles}) } err := db.Run(cmd, nil) if !isNoCmd(err) && user.UserSource != "" && (user.UserSource != "$external" || db.Name != "$external") { @@ -1084,7 +1158,7 @@ func (db *Database) AddUser(username, password string, readOnly bool) error { // RemoveUser removes the authentication credentials of user from the database. func (db *Database) RemoveUser(user string) error { - err := db.Run(bson.D{{"dropUser", user}}, nil) + err := db.Run(bson.D{{Name: "dropUser", Value: user}}, nil) if isNoCmd(err) { users := db.C("system.users") return users.Remove(bson.M{"user": user}) @@ -1098,23 +1172,28 @@ func (db *Database) RemoveUser(user string) error { type indexSpec struct { Name, NS string Key bson.D - Unique bool ",omitempty" - DropDups bool "dropDups,omitempty" - Background bool ",omitempty" - Sparse bool ",omitempty" - Bits int ",omitempty" - Min, Max float64 ",omitempty" - BucketSize float64 "bucketSize,omitempty" - ExpireAfter int "expireAfterSeconds,omitempty" - Weights bson.D ",omitempty" - DefaultLanguage string "default_language,omitempty" - LanguageOverride string "language_override,omitempty" - TextIndexVersion int "textIndexVersion,omitempty" - PartialFilterExpression bson.M "partialFilterExpression,omitempty" - - Collation *Collation "collation,omitempty" -} - + Unique bool `bson:",omitempty"` + DropDups bool `bson:"dropDups,omitempty"` + Background bool `bson:",omitempty"` + Sparse bool `bson:",omitempty"` + Bits int `bson:",omitempty"` + Min, Max float64 `bson:",omitempty"` + BucketSize float64 `bson:"bucketSize,omitempty"` + ExpireAfter int `bson:"expireAfterSeconds,omitempty"` + Weights bson.D `bson:",omitempty"` + DefaultLanguage string `bson:"default_language,omitempty"` + LanguageOverride string `bson:"language_override,omitempty"` + TextIndexVersion int `bson:"textIndexVersion,omitempty"` + PartialFilterExpression bson.M `bson:"partialFilterExpression,omitempty"` + + Collation *Collation `bson:"collation,omitempty"` +} + +// Index are special data structures that store a small portion of the collection’s +// data set in an easy to traverse form. The index stores the value of a specific +// field or set of fields, ordered by the value of the field. The ordering of the +// index entries supports efficient equality matches and range-based query operations. +// In addition, MongoDB can return sorted results by using the ordering in the index. type Index struct { Key []string // Index key fields; prefix name with dash (-) for descending order Unique bool // Prevent two documents from having the same index key @@ -1157,6 +1236,8 @@ type Index struct { Collation *Collation } +// Collation allows users to specify language-specific rules for string comparison, +// such as rules for lettercase and accent marks. type Collation struct { // Locale defines the collation locale. @@ -1271,12 +1352,12 @@ func parseIndexKey(key []string) (*indexKeyInfo, error) { } if kind == "text" { if !isText { - keyInfo.key = append(keyInfo.key, bson.DocElem{"_fts", "text"}, bson.DocElem{"_ftsx", 1}) + keyInfo.key = append(keyInfo.key, bson.DocElem{Name: "_fts", Value: "text"}, bson.DocElem{Name: "_ftsx", Value: 1}) isText = true } - keyInfo.weights = append(keyInfo.weights, bson.DocElem{field, 1}) + keyInfo.weights = append(keyInfo.weights, bson.DocElem{Name: field, Value: 1}) } else { - keyInfo.key = append(keyInfo.key, bson.DocElem{field, order}) + keyInfo.key = append(keyInfo.key, bson.DocElem{Name: field, Value: order}) } } if keyInfo.name == "" { @@ -1436,7 +1517,7 @@ NextField: db := c.Database.With(cloned) // Try with a command first. - err = db.Run(bson.D{{"createIndexes", c.Name}, {"indexes", []indexSpec{spec}}}, nil) + err = db.Run(bson.D{{Name: "createIndexes", Value: c.Name}, {Name: "indexes", Value: []indexSpec{spec}}}, nil) if isNoCmd(err) { // Command not yet supported. Insert into the indexes collection instead. err = db.C("system.indexes").Insert(&spec) @@ -1475,7 +1556,7 @@ func (c *Collection) DropIndex(key ...string) error { ErrMsg string Ok bool }{} - err = db.Run(bson.D{{"dropIndexes", c.Name}, {"index", keyInfo.name}}, &result) + err = db.Run(bson.D{{Name: "dropIndexes", Value: c.Name}, {Name: "index", Value: keyInfo.name}}, &result) if err != nil { return err } @@ -1527,7 +1608,7 @@ func (c *Collection) DropIndexName(name string) error { ErrMsg string Ok bool }{} - err = c.Database.Run(bson.D{{"dropIndexes", c.Name}, {"index", name}}, &result) + err = c.Database.Run(bson.D{{Name: "dropIndexes", Value: c.Name}, {Name: "index", Value: name}}, &result) if err != nil { return err } @@ -1550,7 +1631,7 @@ func (c *Collection) DropAllIndexes() error { ErrMsg string Ok bool }{} - err := db.Run(bson.D{{"dropIndexes", c.Name}, {"index", "*"}}, &result) + err := db.Run(bson.D{{Name: "dropIndexes", Value: c.Name}, {Name: "index", Value: "*"}}, &result) if err != nil { return err } @@ -1563,8 +1644,8 @@ func (c *Collection) DropAllIndexes() error { // nonEventual returns a clone of session and ensures it is not Eventual. // This guarantees that the server that is used for queries may be reused // afterwards when a cursor is received. -func (session *Session) nonEventual() *Session { - cloned := session.Clone() +func (s *Session) nonEventual() *Session { + cloned := s.Clone() if cloned.consistency == Eventual { cloned.SetMode(Monotonic, false) } @@ -1586,7 +1667,7 @@ func (c *Collection) Indexes() (indexes []Index, err error) { Cursor cursorData } var iter *Iter - err = c.Database.With(cloned).Run(bson.D{{"listIndexes", c.Name}, {"cursor", bson.D{{"batchSize", batchSize}}}}, &result) + err = c.Database.With(cloned).Run(bson.D{{Name: "listIndexes", Value: c.Name}, {Name: "cursor", Value: bson.D{{Name: "batchSize", Value: batchSize}}}}, &result) if err == nil { firstBatch := result.Indexes if firstBatch == nil { @@ -1691,7 +1772,7 @@ func simpleIndexKey(realKey bson.D) (key []string) { return } -// ResetIndexCache() clears the cache of previously ensured indexes. +// ResetIndexCache clears the cache of previously ensured indexes. // Following requests to EnsureIndex will contact the server. func (s *Session) ResetIndexCache() { s.cluster().ResetIndexCache() @@ -1744,20 +1825,20 @@ func (s *Session) Clone() *Session { // after it has been closed. func (s *Session) Close() { s.m.Lock() - if s.cluster_ != nil { + if s.mgoCluster != nil { debugf("Closing session %p", s) s.unsetSocket() - s.cluster_.Release() - s.cluster_ = nil + s.mgoCluster.Release() + s.mgoCluster = nil } s.m.Unlock() } func (s *Session) cluster() *mongoCluster { - if s.cluster_ == nil { + if s.mgoCluster == nil { panic("Session already closed") } - return s.cluster_ + return s.mgoCluster } // Refresh puts back any reserved sockets in use and restarts the consistency @@ -1942,7 +2023,7 @@ func (s *Session) SetPrefetch(p float64) { s.m.Unlock() } -// See SetSafe for details on the Safe type. +// Safe session safety mode. See SetSafe for details on the Safe type. type Safe struct { W int // Min # of servers to ack before success WMode string // Write mode for MongoDB 2.0+ (e.g. "majority") @@ -2191,7 +2272,7 @@ func (s *Session) Ping() error { // is established with. If async is true, the call returns immediately, // otherwise it returns after the flush has been made. func (s *Session) Fsync(async bool) error { - return s.Run(bson.D{{"fsync", 1}, {"async", async}}, nil) + return s.Run(bson.D{{Name: "fsync", Value: 1}, {Name: "async", Value: async}}, nil) } // FsyncLock locks all writes in the specific server the session is @@ -2220,12 +2301,12 @@ func (s *Session) Fsync(async bool) error { // http://www.mongodb.org/display/DOCS/Backups // func (s *Session) FsyncLock() error { - return s.Run(bson.D{{"fsync", 1}, {"lock", true}}, nil) + return s.Run(bson.D{{Name: "fsync", Value: 1}, {Name: "lock", Value: true}}, nil) } // FsyncUnlock releases the server for writes. See FsyncLock for details. func (s *Session) FsyncUnlock() error { - err := s.Run(bson.D{{"fsyncUnlock", 1}}, nil) + err := s.Run(bson.D{{Name: "fsyncUnlock", Value: 1}}, nil) if isNoCmd(err) { err = s.DB("admin").C("$cmd.sys.unlock").Find(nil).One(nil) // WTF? } @@ -2266,7 +2347,7 @@ func (c *Collection) Find(query interface{}) *Query { type repairCmd struct { RepairCursor string `bson:"repairCursor"` - Cursor *repairCmdCursor ",omitempty" + Cursor *repairCmdCursor `bson:",omitempty"` } type repairCmdCursor struct { @@ -2307,9 +2388,11 @@ func (c *Collection) Repair() *Iter { // // See the Find method for more details. func (c *Collection) FindId(id interface{}) *Query { - return c.Find(bson.D{{"_id", id}}) + return c.Find(bson.D{{Name: "_id", Value: id}}) } +// Pipe is used to run aggregation queries against a +// collection. type Pipe struct { session *Session collection *Collection @@ -2321,9 +2404,9 @@ type Pipe struct { type pipeCmd struct { Aggregate string Pipeline interface{} - Cursor *pipeCmdCursor ",omitempty" - Explain bool ",omitempty" - AllowDisk bool "allowDiskUse,omitempty" + Cursor *pipeCmdCursor `bson:",omitempty"` + Explain bool `bson:",omitempty"` + AllowDisk bool `bson:"allowDiskUse,omitempty"` } type pipeCmdCursor struct { @@ -2552,8 +2635,13 @@ func (p *Pipe) Batch(n int) *Pipe { return p } +// LastError the error status of the preceding write operation on the current connection. +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/reference/command/getLastError/ +// // mgo.v3: Use a single user-visible error type. - type LastError struct { Err string Code, N, Waited int @@ -2571,13 +2659,14 @@ func (err *LastError) Error() string { } type queryError struct { - Err string "$err" + Err string `bson:"$err"` ErrMsg string Assertion string Code int - AssertionCode int "assertionCode" + AssertionCode int `bson:"assertionCode"` } +// QueryError is returned when a query fails type QueryError struct { Code int Message string @@ -2652,7 +2741,7 @@ func (c *Collection) Update(selector interface{}, update interface{}) error { // // See the Update method for more details. func (c *Collection) UpdateId(id interface{}, update interface{}) error { - return c.Update(bson.D{{"_id", id}}, update) + return c.Update(bson.D{{Name: "_id", Value: id}}, update) } // ChangeInfo holds details about the outcome of an update operation. @@ -2747,7 +2836,7 @@ func (c *Collection) Upsert(selector interface{}, update interface{}) (info *Cha // // See the Upsert method for more details. func (c *Collection) UpsertId(id interface{}, update interface{}) (info *ChangeInfo, err error) { - return c.Upsert(bson.D{{"_id", id}}, update) + return c.Upsert(bson.D{{Name: "_id", Value: id}}, update) } // Remove finds a single document matching the provided selector document @@ -2777,7 +2866,7 @@ func (c *Collection) Remove(selector interface{}) error { // // See the Remove method for more details. func (c *Collection) RemoveId(id interface{}) error { - return c.Remove(bson.D{{"_id", id}}) + return c.Remove(bson.D{{Name: "_id", Value: id}}) } // RemoveAll finds all documents matching the provided selector document @@ -2802,12 +2891,12 @@ func (c *Collection) RemoveAll(selector interface{}) (info *ChangeInfo, err erro // DropDatabase removes the entire database including all of its collections. func (db *Database) DropDatabase() error { - return db.Run(bson.D{{"dropDatabase", 1}}, nil) + return db.Run(bson.D{{Name: "dropDatabase", Value: 1}}, nil) } // DropCollection removes the entire collection including all of its documents. func (c *Collection) DropCollection() error { - return c.Database.Run(bson.D{{"drop", c.Name}}, nil) + return c.Database.Run(bson.D{{Name: "drop", Value: c.Name}}, nil) } // The CollectionInfo type holds metadata about a collection. @@ -2875,37 +2964,37 @@ type CollectionInfo struct { // func (c *Collection) Create(info *CollectionInfo) error { cmd := make(bson.D, 0, 4) - cmd = append(cmd, bson.DocElem{"create", c.Name}) + cmd = append(cmd, bson.DocElem{Name: "create", Value: c.Name}) if info.Capped { if info.MaxBytes < 1 { return fmt.Errorf("Collection.Create: with Capped, MaxBytes must also be set") } - cmd = append(cmd, bson.DocElem{"capped", true}) - cmd = append(cmd, bson.DocElem{"size", info.MaxBytes}) + cmd = append(cmd, bson.DocElem{Name: "capped", Value: true}) + cmd = append(cmd, bson.DocElem{Name: "size", Value: info.MaxBytes}) if info.MaxDocs > 0 { - cmd = append(cmd, bson.DocElem{"max", info.MaxDocs}) + cmd = append(cmd, bson.DocElem{Name: "max", Value: info.MaxDocs}) } } if info.DisableIdIndex { - cmd = append(cmd, bson.DocElem{"autoIndexId", false}) + cmd = append(cmd, bson.DocElem{Name: "autoIndexId", Value: false}) } if info.ForceIdIndex { - cmd = append(cmd, bson.DocElem{"autoIndexId", true}) + cmd = append(cmd, bson.DocElem{Name: "autoIndexId", Value: true}) } if info.Validator != nil { - cmd = append(cmd, bson.DocElem{"validator", info.Validator}) + cmd = append(cmd, bson.DocElem{Name: "validator", Value: info.Validator}) } if info.ValidationLevel != "" { - cmd = append(cmd, bson.DocElem{"validationLevel", info.ValidationLevel}) + cmd = append(cmd, bson.DocElem{Name: "validationLevel", Value: info.ValidationLevel}) } if info.ValidationAction != "" { - cmd = append(cmd, bson.DocElem{"validationAction", info.ValidationAction}) + cmd = append(cmd, bson.DocElem{Name: "validationAction", Value: info.ValidationAction}) } if info.StorageEngine != nil { - cmd = append(cmd, bson.DocElem{"storageEngine", info.StorageEngine}) + cmd = append(cmd, bson.DocElem{Name: "storageEngine", Value: info.StorageEngine}) } if info.Collation != nil { - cmd = append(cmd, bson.DocElem{"collation", info.Collation}) + cmd = append(cmd, bson.DocElem{Name: "collation", Value: info.Collation}) } return c.Database.Run(cmd, nil) @@ -2914,7 +3003,7 @@ func (c *Collection) Create(info *CollectionInfo) error { // Batch sets the batch size used when fetching documents from the database. // It's possible to change this setting on a per-session basis as well, using // the Batch method of Session. - +// // The default batch size is defined by the database itself. As of this // writing, MongoDB will use an initial size of min(100 docs, 4MB) on the // first batch, and 4MB on remaining ones. @@ -3036,9 +3125,9 @@ func (q *Query) Sort(fields ...string) *Query { panic("Sort: empty field name") } if kind == "textScore" { - order = append(order, bson.DocElem{field, bson.M{"$meta": kind}}) + order = append(order, bson.DocElem{Name: field, Value: bson.M{"$meta": kind}}) } else { - order = append(order, bson.DocElem{field, n}) + order = append(order, bson.DocElem{Name: field, Value: n}) } } q.op.options.OrderBy = order @@ -3406,15 +3495,15 @@ func prepareFindOp(socket *mongoSocket, op *queryOp, limit int32) bool { op.hasOptions = false if explain { - op.query = bson.D{{"explain", op.query}} + op.query = bson.D{{Name: "explain", Value: op.query}} return false } return true } type cursorData struct { - FirstBatch []bson.Raw "firstBatch" - NextBatch []bson.Raw "nextBatch" + FirstBatch []bson.Raw `bson:"firstBatch"` + NextBatch []bson.Raw `bson:"nextBatch"` NS string Id int64 } @@ -3477,7 +3566,7 @@ type getMoreCmd struct { func (db *Database) run(socket *mongoSocket, cmd, result interface{}) (err error) { // Database.Run: if name, ok := cmd.(string); ok { - cmd = bson.D{{name, 1}} + cmd = bson.D{{Name: name, Value: 1}} } // Collection.Find: @@ -3566,7 +3655,7 @@ func (db *Database) FindRef(ref *DBRef) *Query { // func (s *Session) FindRef(ref *DBRef) *Query { if ref.Database == "" { - panic(errors.New(fmt.Sprintf("Can't resolve database for %#v", ref))) + panic(fmt.Errorf("Can't resolve database for %#v", ref)) } c := s.DB(ref.Database).C(ref.Collection) return c.FindId(ref.Id) @@ -3587,7 +3676,7 @@ func (db *Database) CollectionNames() (names []string, err error) { Collections []bson.Raw Cursor cursorData } - err = db.With(cloned).Run(bson.D{{"listCollections", 1}, {"cursor", bson.D{{"batchSize", batchSize}}}}, &result) + err = db.With(cloned).Run(bson.D{{Name: "listCollections", Value: 1}, {Name: "cursor", Value: bson.D{{Name: "batchSize", Value: batchSize}}}}, &result) if err == nil { firstBatch := result.Collections if firstBatch == nil { @@ -4052,13 +4141,13 @@ func (q *Query) All(result interface{}) error { return q.Iter().All(result) } -// The For method is obsolete and will be removed in a future release. +// For method is obsolete and will be removed in a future release. // See Iter as an elegant replacement. func (q *Query) For(result interface{}, f func() error) error { return q.Iter().For(result, f) } -// The For method is obsolete and will be removed in a future release. +// For method is obsolete and will be removed in a future release. // See Iter as an elegant replacement. func (iter *Iter) For(result interface{}, f func() error) (err error) { valid := false @@ -4177,8 +4266,8 @@ func (iter *Iter) getMoreCmd() *queryOp { type countCmd struct { Count string Query interface{} - Limit int32 ",omitempty" - Skip int32 ",omitempty" + Limit int32 `bson:",omitempty"` + Skip int32 `bson:",omitempty"` Hint bson.D `bson:"hint,omitempty"` MaxTimeMS int `bson:"maxTimeMS,omitempty"` } @@ -4217,9 +4306,9 @@ func (c *Collection) Count() (n int, err error) { } type distinctCmd struct { - Collection string "distinct" + Collection string `bson:"distinct"` Key string - Query interface{} ",omitempty" + Query interface{} `bson:",omitempty"` } // Distinct unmarshals into result the list of distinct values for the given key. @@ -4256,28 +4345,34 @@ func (q *Query) Distinct(key string, result interface{}) error { } type mapReduceCmd struct { - Collection string "mapreduce" - Map string ",omitempty" - Reduce string ",omitempty" - Finalize string ",omitempty" + Collection string `bson:"mapreduce"` + Map string `bson:",omitempty"` + Reduce string `bson:",omitempty"` + Finalize string `bson:",omitempty"` Out interface{} - Query interface{} ",omitempty" - Sort interface{} ",omitempty" - Scope interface{} ",omitempty" - Limit int32 ",omitempty" - Verbose bool ",omitempty" + Query interface{} `bson:",omitempty"` + Sort interface{} `bson:",omitempty"` + Scope interface{} `bson:",omitempty"` + Limit int32 `bson:",omitempty"` + Verbose bool `bson:",omitempty"` } type mapReduceResult struct { Results bson.Raw Result bson.Raw - TimeMillis int64 "timeMillis" + TimeMillis int64 `bson:"timeMillis"` Counts struct{ Input, Emit, Output int } Ok bool Err string Timing *MapReduceTime } +// MapReduce used to perform Map Reduce operations +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/core/map-reduce/ +// type MapReduce struct { Map string // Map Javascript function code (required) Reduce string // Reduce Javascript function code (required) @@ -4287,6 +4382,7 @@ type MapReduce struct { Verbose bool } +// MapReduceInfo stores informations on a MapReduce operation type MapReduceInfo struct { InputCount int // Number of documents mapped EmitCount int // Number of times reduce called emit @@ -4297,10 +4393,11 @@ type MapReduceInfo struct { VerboseTime *MapReduceTime // Only defined if Verbose was true } +// MapReduceTime stores execution time of a MapReduce operation type MapReduceTime struct { Total int64 // Total time, in nanoseconds - Map int64 "mapTime" // Time within map function, in nanoseconds - EmitLoop int64 "emitLoop" // Time within the emit/map loop, in nanoseconds + Map int64 `bson:"mapTime"` // Time within map function, in nanoseconds + EmitLoop int64 `bson:"emitLoop"` // Time within the emit/map loop, in nanoseconds } // MapReduce executes a map/reduce job for documents covered by the query. @@ -4391,7 +4488,7 @@ func (q *Query) MapReduce(job *MapReduce, result interface{}) (info *MapReduceIn } if cmd.Out == nil { - cmd.Out = bson.D{{"inline", 1}} + cmd.Out = bson.D{{Name: "inline", Value: 1}} } var doc mapReduceResult @@ -4476,14 +4573,14 @@ type Change struct { } type findModifyCmd struct { - Collection string "findAndModify" - Query, Update, Sort, Fields interface{} ",omitempty" - Upsert, Remove, New bool ",omitempty" + Collection string `bson:"findAndModify"` + Query, Update, Sort, Fields interface{} `bson:",omitempty"` + Upsert, Remove, New bool `bson:",omitempty"` } type valueResult struct { Value bson.Raw - LastError LastError "lastErrorObject" + LastError LastError `bson:"lastErrorObject"` } // Apply runs the findAndModify MongoDB command, which allows updating, upserting @@ -4615,7 +4712,7 @@ func (bi *BuildInfo) VersionAtLeast(version ...int) bool { // BuildInfo retrieves the version and other details about the // running MongoDB server. func (s *Session) BuildInfo() (info BuildInfo, err error) { - err = s.Run(bson.D{{"buildInfo", "1"}}, &info) + err = s.Run(bson.D{{Name: "buildInfo", Value: "1"}}, &info) if len(info.VersionArray) == 0 { for _, a := range strings.Split(info.Version, ".") { i, err := strconv.Atoi(a) @@ -4809,7 +4906,7 @@ type writeCmdResult struct { NModified int `bson:"nModified"` Upserted []struct { Index int - Id interface{} `_id` + Id interface{} `bson:"_id"` } ConcernError writeConcernError `bson:"writeConcernError"` Errors []writeCmdError `bson:"writeErrors"` @@ -5022,7 +5119,7 @@ func (c *Collection) writeOpQuery(socket *mongoSocket, safeOp *queryOp, op inter func (c *Collection) writeOpCommand(socket *mongoSocket, safeOp *queryOp, op interface{}, ordered, bypassValidation bool) (lerr *LastError, err error) { var writeConcern interface{} if safeOp == nil { - writeConcern = bson.D{{"w", 0}} + writeConcern = bson.D{{Name: "w", Value: 0}} } else { writeConcern = safeOp.query.(*getLastError) } @@ -5032,46 +5129,46 @@ func (c *Collection) writeOpCommand(socket *mongoSocket, safeOp *queryOp, op int case *insertOp: // http://docs.mongodb.org/manual/reference/command/insert cmd = bson.D{ - {"insert", c.Name}, - {"documents", op.documents}, - {"writeConcern", writeConcern}, - {"ordered", op.flags&1 == 0}, + {Name: "insert", Value: c.Name}, + {Name: "documents", Value: op.documents}, + {Name: "writeConcern", Value: writeConcern}, + {Name: "ordered", Value: op.flags&1 == 0}, } case *updateOp: // http://docs.mongodb.org/manual/reference/command/update cmd = bson.D{ - {"update", c.Name}, - {"updates", []interface{}{op}}, - {"writeConcern", writeConcern}, - {"ordered", ordered}, + {Name: "update", Value: c.Name}, + {Name: "updates", Value: []interface{}{op}}, + {Name: "writeConcern", Value: writeConcern}, + {Name: "ordered", Value: ordered}, } case bulkUpdateOp: // http://docs.mongodb.org/manual/reference/command/update cmd = bson.D{ - {"update", c.Name}, - {"updates", op}, - {"writeConcern", writeConcern}, - {"ordered", ordered}, + {Name: "update", Value: c.Name}, + {Name: "updates", Value: op}, + {Name: "writeConcern", Value: writeConcern}, + {Name: "ordered", Value: ordered}, } case *deleteOp: // http://docs.mongodb.org/manual/reference/command/delete cmd = bson.D{ - {"delete", c.Name}, - {"deletes", []interface{}{op}}, - {"writeConcern", writeConcern}, - {"ordered", ordered}, + {Name: "delete", Value: c.Name}, + {Name: "deletes", Value: []interface{}{op}}, + {Name: "writeConcern", Value: writeConcern}, + {Name: "ordered", Value: ordered}, } case bulkDeleteOp: // http://docs.mongodb.org/manual/reference/command/delete cmd = bson.D{ - {"delete", c.Name}, - {"deletes", op}, - {"writeConcern", writeConcern}, - {"ordered", ordered}, + {Name: "delete", Value: c.Name}, + {Name: "deletes", Value: op}, + {Name: "writeConcern", Value: writeConcern}, + {Name: "ordered", Value: ordered}, } } if bypassValidation { - cmd = append(cmd, bson.DocElem{"bypassDocumentValidation", true}) + cmd = append(cmd, bson.DocElem{Name: "bypassDocumentValidation", Value: true}) } var result writeCmdResult diff --git a/session_test.go b/session_test.go index a4cc04f01..227052719 100644 --- a/session_test.go +++ b/session_test.go @@ -173,10 +173,10 @@ func (s *S) TestURLReadPreferenceTags(c *C) { } tests := []test{ - {"localhost:40001?readPreference=secondary&readPreferenceTags=dc:ny,rack:1", []bson.D{{{"dc", "ny"}, {"rack", "1"}}}}, - {"localhost:40001?readPreference=secondary&readPreferenceTags= dc : ny , rack : 1 ", []bson.D{{{"dc", "ny"}, {"rack", "1"}}}}, - {"localhost:40001?readPreference=secondary&readPreferenceTags=dc:ny", []bson.D{{{"dc", "ny"}}}}, - {"localhost:40001?readPreference=secondary&readPreferenceTags=rack:1&readPreferenceTags=dc:ny", []bson.D{{{"rack", "1"}}, {{"dc", "ny"}}}}, + {"localhost:40001?readPreference=secondary&readPreferenceTags=dc:ny,rack:1", []bson.D{{{Name: "dc", Value: "ny"}, {Name: "rack", Value: "1"}}}}, + {"localhost:40001?readPreference=secondary&readPreferenceTags= dc : ny , rack : 1 ", []bson.D{{{Name: "dc", Value: "ny"}, {Name: "rack", Value: "1"}}}}, + {"localhost:40001?readPreference=secondary&readPreferenceTags=dc:ny", []bson.D{{{Name: "dc", Value: "ny"}}}}, + {"localhost:40001?readPreference=secondary&readPreferenceTags=rack:1&readPreferenceTags=dc:ny", []bson.D{{{Name: "rack", Value: "1"}}, {{Name: "dc", Value: "ny"}}}}, } for _, test := range tests { @@ -211,7 +211,7 @@ func (s *S) TestURLWithAppName(c *C) { db := session.DB("mydb") - err = db.Run(bson.D{{"profile", 2}}, nil) + err = db.Run(bson.D{{Name: "profile", Value: 2}}, nil) c.Assert(err, IsNil) coll := db.C("mycoll") @@ -229,8 +229,8 @@ func (s *S) TestURLWithAppName(c *C) { err = db.C("system.profile").Find(nil).Sort("-ts").One(&profileResult) c.Assert(err, IsNil) c.Assert(appName, Equals, profileResult.AppName) - // reset profiling to 0 as it add unecessary overhead to all other test - err = db.Run(bson.D{{"profile", 0}}, nil) + // reset profiling to 0 as it add unnecessary overhead to all other test + err = db.Run(bson.D{{Name: "profile", Value: 0}}, nil) c.Assert(err, IsNil) } @@ -453,7 +453,7 @@ func (s *S) TestInlineMap(c *C) { var v, result1 struct { A int - M map[string]int ",inline" + M map[string]int `bson:",inline"` } v.A = 1 @@ -574,11 +574,11 @@ func (s *S) TestUpsert(c *C) { ns := []int{40, 41, 42, 43, 44, 45, 46} for _, n := range ns { - err := coll.Insert(bson.D{{"k", n}, {"n", n}}) + err := coll.Insert(bson.D{{Name: "k", Value: n}, {Name: "n", Value: n}}) c.Assert(err, IsNil) } - info, err := coll.Upsert(M{"k": 42}, bson.D{{"k", 42}, {"n", 24}}) + info, err := coll.Upsert(M{"k": 42}, bson.D{{Name: "k", Value: 42}, {Name: "n", Value: 24}}) c.Assert(err, IsNil) c.Assert(info.Updated, Equals, 1) c.Assert(info.Matched, Equals, 1) @@ -590,7 +590,7 @@ func (s *S) TestUpsert(c *C) { c.Assert(result["n"], Equals, 24) // Match but do not change. - info, err = coll.Upsert(M{"k": 42}, bson.D{{"k", 42}, {"n", 24}}) + info, err = coll.Upsert(M{"k": 42}, bson.D{{Name: "k", Value: 42}, {Name: "n", Value: 24}}) c.Assert(err, IsNil) c.Assert(info.Updated, Equals, 1) // On 2.6+ this feels like a server mistake. c.Assert(info.Matched, Equals, 1) @@ -963,6 +963,7 @@ func (s *S) TestCreateCollectionForceIndex(c *C) { c.Assert(err, IsNil) indexes, err := coll.Indexes() + c.Assert(err, IsNil) c.Assert(indexes, HasLen, 1) } @@ -1008,7 +1009,7 @@ func (s *S) TestCreateCollectionValidator(c *C) { err = coll.Create(info) err = coll.Insert(M{"a": 1}) c.Assert(err, IsNil) - err = db.Run(bson.D{{"collMod", "mycoll"}, {"validator", M{"b": M{"$exists": true}}}}, nil) + err = db.Run(bson.D{{Name: "collMod", Value: "mycoll"}, {Name: "validator", Value: M{"b": M{"$exists": true}}}}, nil) c.Assert(err, IsNil) err = coll.Insert(M{"a": 2}) c.Assert(err, ErrorMatches, "Document failed validation") @@ -1910,7 +1911,7 @@ func (s *S) TestResumeIter(c *C) { c.Assert(iter.Err(), IsNil) c.Assert(got.N, Equals, 0) - // Test state returns the cursor ID, and firstBatch + // Test state returns the cursor Id, and firstBatch id, batch := iter.State() c.Assert(id, Not(Equals), 0) c.Assert(len(batch), Equals, 1) @@ -1933,7 +1934,7 @@ func (s *S) TestResumeIter(c *C) { // Done returns true c.Assert(newIter.Done(), Equals, true) - // Ensure state reports no data, no cursor ID + // Ensure state reports no data, no cursor Id id, batch = newIter.State() c.Assert(id, Equals, int64(0)) c.Assert(len(batch), Equals, 0) @@ -1950,7 +1951,7 @@ func (s *S) TestFindIterCursorTimeout(c *C) { defer session.Close() type Doc struct { - Id int "_id" + Id int `bson:"_id"` } coll := session.DB("test").C("test") @@ -1994,7 +1995,7 @@ func (s *S) TestFindIterCursorNoTimeout(c *C) { session.SetCursorTimeout(0) type Doc struct { - Id int "_id" + Id int `bson:"_id"` } coll := session.DB("test").C("test") @@ -2049,7 +2050,7 @@ func (s *S) TestTooManyItemsLimitBug(c *C) { for i := 0; i < 5; i++ { words = append(words, words...) } - doc := bson.D{{"words", words}} + doc := bson.D{{Name: "words", Value: words}} inserts := 10000 limit := 5000 iters := 0 @@ -2085,7 +2086,7 @@ func (s *S) TestBatchSizeZeroGetMore(c *C) { for i := 0; i < 5; i++ { words = append(words, words...) } - doc := bson.D{{"words", words}} + doc := bson.D{{Name: "words", Value: words}} inserts := 10000 iters := 0 for i := 0; i < inserts; i++ { @@ -2316,7 +2317,7 @@ func (s *S) TestFindTailTimeoutWithSleep(c *C) { cresult := struct{ ErrMsg string }{} db := session.DB("mydb") - err = db.Run(bson.D{{"create", "mycoll"}, {"capped", true}, {"size", 1024}}, &cresult) + err = db.Run(bson.D{{Name: "create", Value: "mycoll"}, {Name: "capped", Value: true}, {Name: "size", Value: 1024}}, &cresult) c.Assert(err, IsNil) c.Assert(cresult.ErrMsg, Equals, "") coll := db.C("mycoll") @@ -2410,7 +2411,7 @@ func (s *S) TestFindTailTimeoutNoSleep(c *C) { cresult := struct{ ErrMsg string }{} db := session.DB("mydb") - err = db.Run(bson.D{{"create", "mycoll"}, {"capped", true}, {"size", 1024}}, &cresult) + err = db.Run(bson.D{{Name: "create", Value: "mycoll"}, {Name: "capped", Value: true}, {Name: "size", Value: 1024}}, &cresult) c.Assert(err, IsNil) c.Assert(cresult.ErrMsg, Equals, "") coll := db.C("mycoll") @@ -2495,7 +2496,7 @@ func (s *S) TestFindTailNoTimeout(c *C) { cresult := struct{ ErrMsg string }{} db := session.DB("mydb") - err = db.Run(bson.D{{"create", "mycoll"}, {"capped", true}, {"size", 1024}}, &cresult) + err = db.Run(bson.D{{Name: "create", Value: "mycoll"}, {Name: "capped", Value: true}, {Name: "size", Value: 1024}}, &cresult) c.Assert(err, IsNil) c.Assert(cresult.ErrMsg, Equals, "") coll := db.C("mycoll") @@ -2872,7 +2873,7 @@ func (s *S) TestFindIterSnapshot(c *C) { seen := map[int]bool{} result := struct { - Id int "_id" + Id int `bson:"_id"` }{} for iter.Next(&result) { if len(seen) == 2 { @@ -3014,7 +3015,7 @@ func (s *S) TestPrefetching(c *C) { mgo.SetDebug(false) docs := make([]interface{}, total) for i := 0; i != total; i++ { - docs[i] = bson.D{{"n", i}} + docs[i] = bson.D{{Name: "n", Value: i}} } err = coll.Insert(docs...) c.Assert(err, IsNil) @@ -3959,13 +3960,13 @@ func (s *S) TestEnsureIndexEvalGetIndexes(c *C) { coll := session.DB("mydb").C("mycoll") - err = session.Run(bson.D{{"eval", "db.getSiblingDB('mydb').mycoll.ensureIndex({b: -1})"}}, nil) + err = session.Run(bson.D{{Name: "eval", Value: "db.getSiblingDB('mydb').mycoll.ensureIndex({b: -1})"}}, nil) c.Assert(err, IsNil) - err = session.Run(bson.D{{"eval", "db.getSiblingDB('mydb').mycoll.ensureIndex({a: 1})"}}, nil) + err = session.Run(bson.D{{Name: "eval", Value: "db.getSiblingDB('mydb').mycoll.ensureIndex({a: 1})"}}, nil) c.Assert(err, IsNil) - err = session.Run(bson.D{{"eval", "db.getSiblingDB('mydb').mycoll.ensureIndex({c: -1, e: 1})"}}, nil) + err = session.Run(bson.D{{Name: "eval", Value: "db.getSiblingDB('mydb').mycoll.ensureIndex({c: -1, e: 1})"}}, nil) c.Assert(err, IsNil) - err = session.Run(bson.D{{"eval", "db.getSiblingDB('mydb').mycoll.ensureIndex({d: '2d'})"}}, nil) + err = session.Run(bson.D{{Name: "eval", Value: "db.getSiblingDB('mydb').mycoll.ensureIndex({d: '2d'})"}}, nil) c.Assert(err, IsNil) indexes, err := coll.Indexes() @@ -4050,7 +4051,7 @@ func (s *S) TestDistinct(c *C) { var result []int err = coll.Find(M{"n": M{"$gt": 2}}).Sort("n").Distinct("n", &result) - + c.Assert(err, IsNil) sort.IntSlice(result).Sort() c.Assert(result, DeepEquals, []int{3, 4, 6}) } @@ -4071,7 +4072,7 @@ func (s *S) TestMapReduce(c *C) { Reduce: "function(key, values) { return Array.sum(values); }", } var result []struct { - Id int "_id" + Id int `bson:"_id"` Value int } @@ -4107,7 +4108,7 @@ func (s *S) TestMapReduceFinalize(c *C) { Finalize: "function(key, count) { return {count: count} }", } var result []struct { - Id int "_id" + Id int `bson:"_id"` Value struct{ Count int } } _, err = coll.Find(nil).MapReduce(job, &result) @@ -4148,7 +4149,7 @@ func (s *S) TestMapReduceToCollection(c *C) { expected := map[int]int{1: 1, 2: 2, 3: 1, 4: 2, 6: 1} var item *struct { - Id int "_id" + Id int `bson:"_id"` Value int } mr := session.DB("mydb").C("mr") @@ -4175,7 +4176,7 @@ func (s *S) TestMapReduceToOtherDb(c *C) { job := &mgo.MapReduce{ Map: "function() { emit(this.n, 1); }", Reduce: "function(key, values) { return Array.sum(values); }", - Out: bson.D{{"replace", "mr"}, {"db", "otherdb"}}, + Out: bson.D{{Name: "replace", Value: "mr"}, {Name: "db", Value: "otherdb"}}, } info, err := coll.Find(nil).MapReduce(job, nil) @@ -4188,7 +4189,7 @@ func (s *S) TestMapReduceToOtherDb(c *C) { expected := map[int]int{1: 1, 2: 2, 3: 1, 4: 2, 6: 1} var item *struct { - Id int "_id" + Id int `bson:"_id"` Value int } mr := session.DB("otherdb").C("mr") @@ -4241,6 +4242,7 @@ func (s *S) TestMapReduceScope(c *C) { var result []bson.M _, err = coll.Find(nil).MapReduce(job, &result) + c.Assert(err, IsNil) c.Assert(len(result), Equals, 1) c.Assert(result[0]["value"], Equals, 42.0) } @@ -4404,7 +4406,12 @@ func (s *S) TestRepairCursor(c *C) { coll := session.DB("mydb").C("mycoll3") err = coll.DropCollection() - + if s.versionAtLeast(3, 0) && !s.versionAtLeast(3, 2) { + c.Assert(err.(*mgo.QueryError).Code, Equals, 0) + } else { + c.Assert(err.(*mgo.QueryError).Code, Equals, 26) + c.Assert(err.(*mgo.QueryError).Message, Equals, "ns not found") + } ns := []int{0, 10, 20, 30, 40, 50} for _, n := range ns { coll.Insert(M{"n": n}) @@ -4624,7 +4631,7 @@ func (s *S) TestFindIterDoneWithBatches(c *C) { result := struct{ N int }{} for i := 2; i < 7; i++ { // first check will be with pending local record; - // second will be with open cursor ID but no local + // second will be with open cursor Id but no local // records c.Assert(iter.Done(), Equals, false) ok := iter.Next(&result) @@ -4694,6 +4701,7 @@ func (s *S) TestSetCursorTimeout(c *C) { coll := session.DB("mydb").C("mycoll") err = coll.Insert(M{"n": 42}) + c.Assert(err, IsNil) // This is just a smoke test. Won't wait 10 minutes for an actual timeout. @@ -4712,9 +4720,10 @@ func (s *S) TestNewIterNoServer(c *C) { defer session.Close() data, err := bson.Marshal(bson.M{"a": 1}) + c.Assert(err, IsNil) coll := session.DB("mydb").C("mycoll") - iter := coll.NewIter(nil, []bson.Raw{{3, data}}, 42, nil) + iter := coll.NewIter(nil, []bson.Raw{{Kind: 3, Data: data}}, 42, nil) var result struct{ A int } ok := iter.Next(&result) @@ -4733,9 +4742,10 @@ func (s *S) TestNewIterNoServerPresetErr(c *C) { defer session.Close() data, err := bson.Marshal(bson.M{"a": 1}) + c.Assert(err, IsNil) coll := session.DB("mydb").C("mycoll") - iter := coll.NewIter(nil, []bson.Raw{{3, data}}, 42, fmt.Errorf("my error")) + iter := coll.NewIter(nil, []bson.Raw{{Kind: 3, Data: data}}, 42, fmt.Errorf("my error")) var result struct{ A int } ok := iter.Next(&result) @@ -4761,8 +4771,8 @@ func (s *S) TestBypassValidation(c *C) { c.Assert(err, IsNil) err = coll.Database.Run(bson.D{ - {"collMod", "mycoll"}, - {"validator", M{"s": M{"$type": "string"}}}, + {Name: "collMod", Value: "mycoll"}, + {Name: "validator", Value: M{"s": M{"$type": "string"}}}, }, nil) c.Assert(err, IsNil) @@ -4870,9 +4880,9 @@ func (s *S) BenchmarkFindIterRaw(c *C) { coll := session.DB("mydb").C("mycoll") doc := bson.D{ - {"f2", "a short string"}, - {"f3", bson.D{{"1", "one"}, {"2", 2.0}}}, - {"f4", []string{"a", "b", "c", "d", "e", "f", "g"}}, + {Name: "f2", Value: "a short string"}, + {Name: "f3", Value: bson.D{{Name: "1", Value: "one"}, {Name: "2", Value: 2.0}}}, + {Name: "f4", Value: []string{"a", "b", "c", "d", "e", "f", "g"}}, } for i := 0; i < c.N+1; i++ { @@ -4903,7 +4913,7 @@ func BenchmarkInsertSingle(b *testing.B) { defer session.Close() doc := bson.D{ - {"A", strings.Repeat("*", 256)}, + {Name: "A", Value: strings.Repeat("*", 256)}, } coll := session.DB("mydb").C("benchmarkcoll") b.ResetTimer() @@ -4925,7 +4935,7 @@ func BenchmarkInsertMultiple(b *testing.B) { docs := make([]interface{}, 100) for i := range docs { docs[i] = bson.D{ - {"A", strings.Repeat("*", 256)}, + {Name: "A", Value: strings.Repeat("*", 256)}, } } coll := session.DB("mydb").C("benchmarkcoll") diff --git a/socket.go b/socket.go index c31c8312b..72fab9cf7 100644 --- a/socket.go +++ b/socket.go @@ -83,16 +83,16 @@ type queryOp struct { } type queryWrapper struct { - Query interface{} "$query" - OrderBy interface{} "$orderby,omitempty" - Hint interface{} "$hint,omitempty" - Explain bool "$explain,omitempty" - Snapshot bool "$snapshot,omitempty" - ReadPreference bson.D "$readPreference,omitempty" - MaxScan int "$maxScan,omitempty" - MaxTimeMS int "$maxTimeMS,omitempty" - Comment string "$comment,omitempty" - Collation *Collation "$collation,omitempty" + Query interface{} `bson:"$query"` + OrderBy interface{} `bson:"$orderby,omitempty"` + Hint interface{} `bson:"$hint,omitempty"` + Explain bool `bson:"$explain,omitempty"` + Snapshot bool `bson:"$snapshot,omitempty"` + ReadPreference bson.D `bson:"$readPreference,omitempty"` + MaxScan int `bson:"$maxScan,omitempty"` + MaxTimeMS int `bson:"$maxTimeMS,omitempty"` + Comment string `bson:"$comment,omitempty"` + Collation *Collation `bson:"$collation,omitempty"` } func (op *queryOp) finalQuery(socket *mongoSocket) interface{} { @@ -116,9 +116,9 @@ func (op *queryOp) finalQuery(socket *mongoSocket) interface{} { } op.hasOptions = true op.options.ReadPreference = make(bson.D, 0, 2) - op.options.ReadPreference = append(op.options.ReadPreference, bson.DocElem{"mode", modeName}) + op.options.ReadPreference = append(op.options.ReadPreference, bson.DocElem{Name: "mode", Value: modeName}) if len(op.serverTags) > 0 { - op.options.ReadPreference = append(op.options.ReadPreference, bson.DocElem{"tags", op.serverTags}) + op.options.ReadPreference = append(op.options.ReadPreference, bson.DocElem{Name: "tags", Value: op.serverTags}) } } if op.hasOptions { diff --git a/stats.go b/stats.go index 59723e60c..dcbd01045 100644 --- a/stats.go +++ b/stats.go @@ -33,6 +33,7 @@ import ( var stats *Stats var statsMutex sync.Mutex +// SetStats enable database state monitoring func SetStats(enabled bool) { statsMutex.Lock() if enabled { @@ -45,6 +46,7 @@ func SetStats(enabled bool) { statsMutex.Unlock() } +// GetStats return the current database state func GetStats() (snapshot Stats) { statsMutex.Lock() snapshot = *stats @@ -52,6 +54,7 @@ func GetStats() (snapshot Stats) { return } +// ResetStats reset Stats to the previous database state func ResetStats() { statsMutex.Lock() debug("Resetting stats") @@ -66,6 +69,13 @@ func ResetStats() { return } +// Stats holds info on the database state +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/reference/command/serverStatus/ +// +// TODO outdated fields ? type Stats struct { Clusters int MasterConns int diff --git a/txn/debug.go b/txn/debug.go index e23d277af..73ae8db3b 100644 --- a/txn/debug.go +++ b/txn/debug.go @@ -11,15 +11,15 @@ import ( var ( debugEnabled bool - logger log_Logger + logger logLogger ) -type log_Logger interface { +type logLogger interface { Output(calldepth int, s string) error } -// Specify the *log.Logger where logged messages should be sent to. -func SetLogger(l log_Logger) { +// SetLogger specify the *log.Logger where logged messages should be sent to. +func SetLogger(l logLogger) { logger = l } @@ -28,6 +28,8 @@ func SetDebug(debug bool) { debugEnabled = debug } +// ErrChaos error returned when operation failed due to +// the failure injection mechanism. var ErrChaos = fmt.Errorf("interrupted by chaos") var debugId uint32 diff --git a/txn/flusher.go b/txn/flusher.go index 43e01f4a9..3d1882d7f 100644 --- a/txn/flusher.go +++ b/txn/flusher.go @@ -35,7 +35,7 @@ type tokenAndId struct { bid bson.ObjectId } -func (ti tokenAndId) id() bson.ObjectId { +func (ti tokenAndId) Id() bson.ObjectId { return ti.bid } @@ -83,7 +83,7 @@ func (f *flusher) run() (err error) { NextPair: for i := 0; i < len(dqueue); i++ { pred := dqueue[i] - predid := pred.id() + predid := pred.Id() predt := seen[predid] if predt == nil || predt.Nonce != pred.nonce() { continue @@ -95,7 +95,7 @@ func (f *flusher) run() (err error) { for j := i + 1; j < len(dqueue); j++ { succ := dqueue[j] - succid := succ.id() + succid := succ.Id() succt := seen[succid] if succt == nil || succt.Nonce != succ.nonce() { continue @@ -142,14 +142,14 @@ func (f *flusher) run() (err error) { if len(scc) == 1 { pull[scc[0]] = seen[scc[0]] } - for _, id := range scc { - if err := f.advance(seen[id], pull, true); err != nil { + for _, Id := range scc { + if err := f.advance(seen[Id], pull, true); err != nil { return err } } if len(scc) > 1 { - for _, id := range scc { - pull[id] = seen[id] + for _, Id := range scc { + pull[Id] = seen[Id] } } } @@ -169,15 +169,13 @@ func (f *flusher) recurse(t *transaction, seen map[bson.ObjectId]*transaction, p remaining := make([]bson.ObjectId, 0, len(f.queue[dkey])) toPreload := make(map[bson.ObjectId]struct{}, len(f.queue[dkey])) for _, dtt := range f.queue[dkey] { - id := dtt.id() - if _, scheduled := toPreload[id]; seen[id] != nil || scheduled || preloaded[id] != nil { + Id := dtt.Id() + if _, scheduled := toPreload[Id]; seen[Id] != nil || scheduled || preloaded[Id] != nil { continue } - toPreload[id] = struct{}{} - remaining = append(remaining, id) + toPreload[Id] = struct{}{} + remaining = append(remaining, Id) } - // done with this map - toPreload = nil for len(remaining) > 0 { batch := remaining if len(batch) > preloadBatchSize { @@ -188,13 +186,13 @@ func (f *flusher) recurse(t *transaction, seen map[bson.ObjectId]*transaction, p if err != nil { return err } - for _, id := range batch { - if seen[id] != nil { + for _, Id := range batch { + if seen[Id] != nil { continue } - qt, ok := preloaded[id] + qt, ok := preloaded[Id] if !ok { - qt, err = f.load(id) + qt, err = f.load(Id) if err != nil { return err } @@ -239,7 +237,6 @@ func (f *flusher) advance(t *transaction, pull map[bson.ObjectId]*transaction, f panic(fmt.Errorf("transaction in unknown state: %q", t.State)) } } - panic("unreachable") } type stash string @@ -264,11 +261,15 @@ const ( stashInserting stashState = "inserting" ) -var txnFields = bson.D{{"txn-queue", 1}, {"txn-revno", 1}, {"txn-remove", 1}, {"txn-insert", 1}} +var txnFields = bson.D{ + {Name: "txn-queue", Value: 1}, + {Name: "txn-revno", Value: 1}, + {Name: "txn-remove", Value: 1}, + {Name: "txn-insert", Value: 1}} var errPreReqs = fmt.Errorf("transaction has pre-requisites and force is false") -// prepare injects t's id onto txn-queue for all affected documents +// prepare injects t's Id onto txn-queue for all affected documents // and collects the current txn-queue and txn-revno values during // the process. If the prepared txn-queue indicates that there are // pre-requisite transactions to be applied and the force parameter @@ -290,7 +291,7 @@ func (f *flusher) prepare(t *transaction, force bool) (revnos []int64, err error NextDoc: for _, dkey := range dkeys { change := mgo.Change{ - Update: bson.D{{"$addToSet", bson.D{{"txn-queue", tt}}}}, + Update: bson.D{{Name: "$addToSet", Value: bson.D{{Name: "txn-queue", Value: tt}}}}, ReturnNew: true, } c := f.tc.Database.C(dkey.C) @@ -303,7 +304,7 @@ NextDoc: if f.opts.MaxTxnQueueLength > 0 && len(info.Queue) > f.opts.MaxTxnQueueLength { // abort with TXN Queue too long, but remove the entry we just added innerErr := c.UpdateId(dkey.Id, - bson.D{{"$pullAll", bson.D{{"txn-queue", []token{tt}}}}}) + bson.D{{Name: "$pullAll", Value: bson.D{{Name: "txn-queue", Value: []token{tt}}}}}) if innerErr != nil { f.debugf("error while backing out of queue-too-long: %v", innerErr) } @@ -387,8 +388,8 @@ NextDoc: // Save the prepared nonce onto t. nonce := tt.nonce() - qdoc := bson.D{{"_id", t.Id}, {"s", tpreparing}} - udoc := bson.D{{"$set", bson.D{{"s", tprepared}, {"n", nonce}}}} + qdoc := bson.D{{Name: "_id", Value: t.Id}, {Name: "s", Value: tpreparing}} + udoc := bson.D{{Name: "$set", Value: bson.D{{Name: "s", Value: tprepared}, {Name: "n", Value: nonce}}}} chaos("set-prepared") err = f.tc.Update(qdoc, udoc) if err == nil { @@ -426,12 +427,12 @@ NextDoc: } func (f *flusher) unstashToken(tt token, dkey docKey) error { - qdoc := bson.D{{"_id", dkey}, {"txn-queue", tt}} - udoc := bson.D{{"$pull", bson.D{{"txn-queue", tt}}}} + qdoc := bson.D{{Name: "_id", Value: dkey}, {Name: "txn-queue", Value: tt}} + udoc := bson.D{{Name: "$pull", Value: bson.D{{Name: "txn-queue", Value: tt}}}} chaos("") if err := f.sc.Update(qdoc, udoc); err == nil { chaos("") - err = f.sc.Remove(bson.D{{"_id", dkey}, {"txn-queue", bson.D{}}}) + err = f.sc.Remove(bson.D{{Name: "_id", Value: dkey}, {Name: "txn-queue", Value: bson.D{}}}) } else if err != mgo.ErrNotFound { return err } @@ -511,15 +512,15 @@ func (f *flusher) rescan(t *transaction, force bool) (revnos []int64, err error) revno[dkey] = info.Revno found := false - for _, id := range info.Queue { - if id == tt { + for _, Id := range info.Queue { + if Id == tt { found = true break } } f.queue[dkey] = tokensWithIds(info.Queue) if !found { - // Rescanned transaction id was not in the queue. This could mean one + // Rescanned transaction Id was not in the queue. This could mean one // of three things: // 1) The transaction was applied and popped by someone else. This is // the common case. @@ -587,7 +588,7 @@ NextDoc: for _, dtt := range f.queue[dkey] { if dtt.tt == tt { continue NextDoc - } else if dtt.id() != ttId { + } else if dtt.Id() != ttId { prereqs = true } } @@ -599,7 +600,7 @@ NextDoc: func (f *flusher) reload(t *transaction) error { var newt transaction query := f.tc.FindId(t.Id) - query.Select(bson.D{{"s", 1}, {"n", 1}, {"r", 1}}) + query.Select(bson.D{{Name: "s", Value: 1}, {Name: "n", Value: 1}, {Name: "r", Value: 1}}) if err := query.One(&newt); err != nil { return fmt.Errorf("failed to reload transaction: %v", err) } @@ -610,8 +611,8 @@ func (f *flusher) reload(t *transaction) error { return nil } -func (f *flusher) loadAndApply(id bson.ObjectId) error { - t, err := f.load(id) +func (f *flusher) loadAndApply(Id bson.ObjectId) error { + t, err := f.load(Id) if err != nil { return err } @@ -643,28 +644,28 @@ func (f *flusher) assert(t *transaction, revnos []int64, pull map[bson.ObjectId] continue } if op.Insert != nil { - return fmt.Errorf("Insert can only Assert txn.DocMissing", op.Assert) + return fmt.Errorf("Insert can only Assert txn.DocMissing, was %v", op.Assert) } // if revnos[i] < 0 { abort }? - qdoc = append(qdoc[:0], bson.DocElem{"_id", op.Id}) + qdoc = append(qdoc[:0], bson.DocElem{Name: "_id", Value: op.Id}) if op.Assert != DocMissing { var revnoq interface{} if n := revno[dkey]; n == 0 { - revnoq = bson.D{{"$exists", false}} + revnoq = bson.D{{Name: "$exists", Value: false}} } else { revnoq = n } // XXX Add tt to the query here, once we're sure it's all working. // Not having it increases the chances of breaking on bad logic. - qdoc = append(qdoc, bson.DocElem{"txn-revno", revnoq}) + qdoc = append(qdoc, bson.DocElem{Name: "txn-revno", Value: revnoq}) if op.Assert != DocExists { - qdoc = append(qdoc, bson.DocElem{"$or", []interface{}{op.Assert}}) + qdoc = append(qdoc, bson.DocElem{Name: "$or", Value: []interface{}{op.Assert}}) } } c := f.tc.Database.C(op.C) - if err := c.Find(qdoc).Select(bson.D{{"_id", 1}}).One(nil); err == mgo.ErrNotFound { + if err := c.Find(qdoc).Select(bson.D{{Name: "_id", Value: 1}}).One(nil); err == mgo.ErrNotFound { // Assertion failed or someone else started applying. return f.abortOrReload(t, revnos, pull) } else if err != nil { @@ -678,8 +679,8 @@ func (f *flusher) assert(t *transaction, revnos []int64, pull map[bson.ObjectId] func (f *flusher) abortOrReload(t *transaction, revnos []int64, pull map[bson.ObjectId]*transaction) (err error) { f.debugf("Aborting or reloading %s (was %q)", t, t.State) if t.State == tprepared { - qdoc := bson.D{{"_id", t.Id}, {"s", tprepared}} - udoc := bson.D{{"$set", bson.D{{"s", taborting}}}} + qdoc := bson.D{{Name: "_id", Value: t.Id}, {Name: "s", Value: tprepared}} + udoc := bson.D{{Name: "$set", Value: bson.D{{Name: "s", Value: taborting}}}} chaos("set-aborting") if err = f.tc.Update(qdoc, udoc); err == nil { t.State = taborting @@ -711,7 +712,7 @@ func (f *flusher) abortOrReload(t *transaction, revnos []int64, pull map[bson.Ob if len(pullAll) == 0 { continue } - udoc := bson.D{{"$pullAll", bson.D{{"txn-queue", pullAll}}}} + udoc := bson.D{{Name: "$pullAll", Value: bson.D{{Name: "txn-queue", Value: pullAll}}}} chaos("") if revnos[i] < 0 { err = f.sc.UpdateId(dkey, udoc) @@ -724,7 +725,7 @@ func (f *flusher) abortOrReload(t *transaction, revnos []int64, pull map[bson.Ob } } } - udoc := bson.D{{"$set", bson.D{{"s", taborted}}}} + udoc := bson.D{{Name: "$set", Value: bson.D{{Name: "s", Value: taborted}}}} chaos("set-aborted") if err := f.tc.UpdateId(t.Id, udoc); err != nil && err != mgo.ErrNotFound { return err @@ -746,8 +747,8 @@ func (f *flusher) checkpoint(t *transaction, revnos []int64) error { } // Save in t the txn-revno values the transaction must run on. - qdoc := bson.D{{"_id", t.Id}, {"s", tprepared}} - udoc := bson.D{{"$set", bson.D{{"s", tapplying}, {"r", revnos}}}} + qdoc := bson.D{{Name: "_id", Value: t.Id}, {Name: "s", Value: tprepared}} + udoc := bson.D{{Name: "$set", Value: bson.D{{Name: "s", Value: tapplying}, {Name: "r", Value: revnos}}}} chaos("set-applying") err := f.tc.Update(qdoc, udoc) if err == nil { @@ -771,7 +772,7 @@ func (f *flusher) apply(t *transaction, pull map[bson.ObjectId]*transaction) err } logRevnos := append([]int64(nil), t.Revnos...) - logDoc := bson.D{{"_id", t.Id}} + logDoc := bson.D{{Name: "_id", Value: t.Id}} tt := tokenFor(t) for i := range t.Ops { @@ -788,18 +789,18 @@ func (f *flusher) apply(t *transaction, pull map[bson.ObjectId]*transaction) err c := f.tc.Database.C(op.C) - qdoc := bson.D{{"_id", dkey.Id}, {"txn-revno", revno}, {"txn-queue", tt}} + qdoc := bson.D{{Name: "_id", Value: dkey.Id}, {Name: "txn-revno", Value: revno}, {Name: "txn-queue", Value: tt}} if op.Insert != nil { qdoc[0].Value = dkey if revno == -1 { - qdoc[1].Value = bson.D{{"$exists", false}} + qdoc[1].Value = bson.D{{Name: "$exists", Value: false}} } } else if revno == 0 { // There's no document with revno 0. The only way to see it is // when an existent document participates in a transaction the // first time. Txn-inserted documents get revno -1 while in the // stash for the first time, and -revno-1 == 2 when they go live. - qdoc[1].Value = bson.D{{"$exists", false}} + qdoc[1].Value = bson.D{{Name: "$exists", Value: false}} } pullAll := tokensToPull(dqueue, pull, tt) @@ -818,10 +819,10 @@ func (f *flusher) apply(t *transaction, pull map[bson.ObjectId]*transaction) err if d, err = objToDoc(op.Update); err != nil { return err } - if d, err = addToDoc(d, "$pullAll", bson.D{{"txn-queue", pullAll}}); err != nil { + if d, err = addToDoc(d, "$pullAll", bson.D{{Name: "txn-queue", Value: pullAll}}); err != nil { return err } - if d, err = addToDoc(d, "$set", bson.D{{"txn-revno", newRevno}}); err != nil { + if d, err = addToDoc(d, "$set", bson.D{{Name: "txn-revno", Value: newRevno}}); err != nil { return err } chaos("") @@ -836,7 +837,7 @@ func (f *flusher) apply(t *transaction, pull map[bson.ObjectId]*transaction) err nonce := newNonce() stash := txnInfo{} change := mgo.Change{ - Update: bson.D{{"$push", bson.D{{"n", nonce}}}}, + Update: bson.D{{Name: "$push", Value: bson.D{{Name: "n", Value: nonce}}}}, Upsert: true, ReturnNew: true, } @@ -844,7 +845,7 @@ func (f *flusher) apply(t *transaction, pull map[bson.ObjectId]*transaction) err return err } change = mgo.Change{ - Update: bson.D{{"$set", bson.D{{"txn-remove", t.Id}}}}, + Update: bson.D{{Name: "$set", Value: bson.D{{Name: "txn-remove", Value: t.Id}}}}, ReturnNew: true, } var info txnInfo @@ -858,14 +859,14 @@ func (f *flusher) apply(t *transaction, pull map[bson.ObjectId]*transaction) err var set, unset bson.D if revno == 0 { // Missing revno in stash means -1. - set = bson.D{{"txn-queue", info.Queue}} - unset = bson.D{{"n", 1}, {"txn-revno", 1}} + set = bson.D{{Name: "txn-queue", Value: info.Queue}} + unset = bson.D{{Name: "n", Value: 1}, {Name: "txn-revno", Value: 1}} } else { - set = bson.D{{"txn-queue", info.Queue}, {"txn-revno", newRevno}} - unset = bson.D{{"n", 1}} + set = bson.D{{Name: "txn-queue", Value: info.Queue}, {Name: "txn-revno", Value: newRevno}} + unset = bson.D{{Name: "n", Value: 1}} } - qdoc := bson.D{{"_id", dkey}, {"n", nonce}} - udoc := bson.D{{"$set", set}, {"$unset", unset}} + qdoc := bson.D{{Name: "_id", Value: dkey}, {Name: "n", Value: nonce}} + udoc := bson.D{{Name: "$set", Value: set}, {Name: "$unset", Value: unset}} if err = f.sc.Update(qdoc, udoc); err == nil { updated = true } else if err != mgo.ErrNotFound { @@ -890,14 +891,14 @@ func (f *flusher) apply(t *transaction, pull map[bson.ObjectId]*transaction) err return err } change := mgo.Change{ - Update: bson.D{{"$set", bson.D{{"txn-insert", t.Id}}}}, + Update: bson.D{{Name: "$set", Value: bson.D{{Name: "txn-insert", Value: t.Id}}}}, ReturnNew: true, } chaos("") var info txnInfo if _, err = f.sc.Find(qdoc).Apply(change, &info); err == nil { f.debugf("Stash for document %v has revno %d and queue: %v", dkey, info.Revno, info.Queue) - d = setInDoc(d, bson.D{{"_id", op.Id}, {"txn-revno", newRevno}, {"txn-queue", info.Queue}}) + d = setInDoc(d, bson.D{{Name: "_id", Value: op.Id}, {Name: "txn-revno", Value: newRevno}, {Name: "txn-queue", Value: info.Queue}}) // Unlikely yet unfortunate race in here if this gets seriously // delayed. If someone inserts+removes meanwhile, this will // reinsert, and there's no way to avoid that while keeping the @@ -947,7 +948,7 @@ func (f *flusher) apply(t *transaction, pull map[bson.ObjectId]*transaction) err } } if dr == nil { - logDoc = append(logDoc, bson.DocElem{op.C, bson.D{{"d", []interface{}{}}, {"r", []int64{}}}}) + logDoc = append(logDoc, bson.DocElem{Name: op.C, Value: bson.D{{Name: "d", Value: []interface{}{}}, {Name: "r", Value: []int64{}}}}) dr = logDoc[len(logDoc)-1].Value.(bson.D) } dr[0].Value = append(dr[0].Value.([]interface{}), op.Id) @@ -971,7 +972,9 @@ func (f *flusher) apply(t *transaction, pull map[bson.ObjectId]*transaction) err // it has been applied and mark it at such. f.debugf("Marking %s as applied", t) chaos("set-applied") - f.tc.Update(bson.D{{"_id", t.Id}, {"s", tapplying}}, bson.D{{"$set", bson.D{{"s", tapplied}}}}) + f.tc.Update( + bson.D{{Name: "_id", Value: t.Id}, {Name: "s", Value: tapplying}}, + bson.D{{Name: "$set", Value: bson.D{{Name: "s", Value: tapplied}}}}) return nil } @@ -982,7 +985,7 @@ func tokensToPull(dqueue []tokenAndId, pull map[bson.ObjectId]*transaction, dont if dtt.tt == dontPull { continue } - if _, ok := pull[dtt.id()]; ok { + if _, ok := pull[dtt.Id()]; ok { // It was handled before and this is a leftover invalid // nonce in the queue. Cherry-pick it out. result = append(result, dtt.tt) @@ -1009,14 +1012,14 @@ func addToDoc(doc bson.D, key string, add bson.D) (bson.D, error) { if elem.Name != key { continue } - if old, ok := elem.Value.(bson.D); ok { - elem.Value = append(old, add...) - return doc, nil - } else { + old, ok := elem.Value.(bson.D) + if !ok { return nil, fmt.Errorf("invalid %q value in change document: %#v", key, elem.Value) } + elem.Value = append(old, add...) + return doc, nil } - return append(doc, bson.DocElem{key, add}), nil + return append(doc, bson.DocElem{Name: key, Value: add}), nil } func setInDoc(doc bson.D, set bson.D) bson.D { diff --git a/txn/txn.go b/txn/txn.go index 8b44c8339..6e5c89f34 100644 --- a/txn/txn.go +++ b/txn/txn.go @@ -1,4 +1,4 @@ -// The txn package implements support for multi-document transactions. +// Package txn implements support for multi-document transactions. // // For details check the following blog post: // @@ -113,7 +113,7 @@ NextOp: } // tokenFor returns a unique transaction token that -// is composed by t's id and a nonce. If t already has +// is composed by t's Id and a nonce. If t already has // a nonce assigned to it, it will be used, otherwise // a new nonce will be generated. func tokenFor(t *transaction) token { @@ -207,10 +207,13 @@ func (op *Op) name() string { } const ( - // DocExists and DocMissing may be used on an operation's + // DocExists may be used on an operation's // Assert value to assert that the document with the given - // Id exists or does not exist, respectively. - DocExists = "d+" + // ID exists. + DocExists = "d+" + // DocMissing may be used on an operation's + // Assert value to assert that the document with the given + // ID does not exist. DocMissing = "d-" ) @@ -268,13 +271,15 @@ func DefaultRunnerOptions() RunnerOptions { } } +// ErrAborted error returned if one or more operations +// can't be applied. var ErrAborted = fmt.Errorf("transaction aborted") // Run creates a new transaction with ops and runs it immediately. -// The id parameter specifies the transaction id, and may be written +// The id parameter specifies the transaction Id, and may be written // down ahead of time to later verify the success of the change and // resume it, when the procedure is interrupted for any reason. If -// empty, a random id will be generated. +// empty, a random Id will be generated. // The info parameter, if not nil, is included under the "i" // field of the transaction document. // @@ -291,7 +296,7 @@ var ErrAborted = fmt.Errorf("transaction aborted") // reason, it may be resumed explicitly or by attempting to apply // another transaction on any of the documents targeted by ops, as // long as the interruption was made after the transaction document -// itself was inserted. Run Resume with the obtained transaction id +// itself was inserted. Run Resume with the obtained transaction Id // to confirm whether the transaction was applied or not. // // Any number of transactions may be run concurrently, with one @@ -349,7 +354,7 @@ func (r *Runner) Run(ops []Op, id bson.ObjectId, info interface{}) (err error) { // from individual transactions are ignored. func (r *Runner) ResumeAll() (err error) { debugf("Resuming all unfinished transactions") - iter := r.tc.Find(bson.D{{"s", bson.D{{"$in", []state{tpreparing, tprepared, tapplying}}}}}).Iter() + iter := r.tc.Find(bson.D{{Name: "s", Value: bson.D{{Name: "$in", Value: []state{tpreparing, tprepared, tapplying}}}}}).Iter() var t transaction for iter.Next(&t) { if t.State == tapplied || t.State == taborted { @@ -366,7 +371,7 @@ func (r *Runner) ResumeAll() (err error) { return nil } -// Resume resumes the transaction with id. It returns mgo.ErrNotFound +// Resume resumes the transaction with Id. It returns mgo.ErrNotFound // if the transaction is not found. Otherwise, it has the same semantics // of the Run method after the transaction is inserted. func (r *Runner) Resume(id bson.ObjectId) (err error) { @@ -417,8 +422,8 @@ func (r *Runner) PurgeMissing(collections ...string) error { type S []interface{} type TDoc struct { - Id interface{} "_id" - TxnQueue []string "txn-queue" + Id interface{} `bson:"_id"` + TxnQueue []string `bson:"txn-queue"` } found := make(map[bson.ObjectId]bool) @@ -451,8 +456,8 @@ func (r *Runner) PurgeMissing(collections ...string) error { } type StashTDoc struct { - Id docKey "_id" - TxnQueue []string "txn-queue" + Id docKey `bson:"_id"` + TxnQueue []string `bson:"txn-queue"` } iter := r.sc.Find(nil).Select(bson.M{"_id": 1, "txn-queue": 1}).Iter() @@ -511,7 +516,6 @@ func (r *Runner) loadMulti(ids []bson.ObjectId, preloaded map[bson.ObjectId]*tra return nil } - type typeNature int const ( @@ -644,7 +648,6 @@ func structcmp(a, b interface{}) int { return 1 } } - panic("unreachable") } func isExported(name string) bool { diff --git a/txn/txn_test.go b/txn/txn_test.go index 8b85986b5..fdc00eb3d 100644 --- a/txn/txn_test.go +++ b/txn/txn_test.go @@ -121,7 +121,7 @@ func (s *S) TestInsert(c *C) { c.Assert(account.Balance, Equals, 200) } -func (s *S) TestInsertStructID(c *C) { +func (s *S) TestInsertStructId(c *C) { type id struct { FirstName string LastName string @@ -374,8 +374,8 @@ func (s *S) TestAssertNestedOr(c *C) { ops := []txn.Op{{ C: "accounts", Id: 0, - Assert: bson.D{{"$or", []bson.D{{{"balance", 100}}, {{"balance", 300}}}}}, - Update: bson.D{{"$inc", bson.D{{"balance", 100}}}}, + Assert: bson.D{{Name: "$or", Value: []bson.D{{{Name: "balance", Value: 100}}, {{Name: "balance", Value: 300}}}}}, + Update: bson.D{{Name: "$inc", Value: bson.D{{Name: "balance", Value: 100}}}}, }} err = s.runner.Run(ops, "", nil) @@ -390,7 +390,7 @@ func (s *S) TestAssertNestedOr(c *C) { func (s *S) TestVerifyFieldOrdering(c *C) { // Used to have a map in certain operations, which means // the ordering of fields would be messed up. - fields := bson.D{{"a", 1}, {"b", 2}, {"c", 3}} + fields := bson.D{{Name: "a", Value: 1}, {Name: "b", Value: 2}, {Name: "c", Value: 3}} ops := []txn.Op{{ C: "accounts", Id: 0, @@ -441,8 +441,8 @@ func (s *S) TestChangeLog(c *C) { type IdList []interface{} type Log struct { - Docs IdList "d" - Revnos []int64 "r" + Docs IdList `bson:"d"` + Revnos []int64 `bson:"r"` } var m map[string]*Log err = chglog.FindId(id).One(&m) @@ -567,7 +567,7 @@ func (s *S) TestPurgeMissing(c *C) { err = s.accounts.FindId(want.Id).One(&got) if want.Balance == -1 { if err != mgo.ErrNotFound { - c.Errorf("Account %d should not exist, find got err=%#v", err) + c.Errorf("Account %d should not exist, find got err=%#v", got, err) } } else if err != nil { c.Errorf("Account %d should have balance of %d, but wasn't found", want.Id, want.Balance) @@ -776,7 +776,7 @@ func (s *S) TestPurgeMissingPipelineSizeLimit(c *C) { // processing the txn-queue fields of stash documents so insert // the large txn-queue there too to ensure that no longer happens. err = s.sc.Insert( - bson.D{{"c", "accounts"}, {"id", 0}}, + bson.D{{Name: "c", Value: "accounts"}, {Name: "id", Value: 0}}, bson.M{"txn-queue": fakeTxnQueue}, ) c.Assert(err, IsNil) @@ -789,7 +789,6 @@ func (s *S) TestPurgeMissingPipelineSizeLimit(c *C) { var flaky = flag.Bool("flaky", false, "Include flaky tests") var txnQueueLength = flag.Int("qlength", 100, "txn-queue length for tests") - func (s *S) TestTxnQueueStressTest(c *C) { // This fails about 20% of the time on Mongo 3.2 (I haven't tried // other versions) with account balance being 3999 instead of @@ -925,11 +924,6 @@ func (s *S) TestTxnQueueBrokenPrepared(c *C) { c.Logf("%8.3fs to set up %d 'prepared' txns", time.Since(t).Seconds(), *txnQueueLength) t = time.Now() s.accounts.UpdateId(0, bson.M{"$pullAll": bson.M{"txn-queue": []string{badTxnToken}}}) - ops = []txn.Op{{ - C: "accounts", - Id: 0, - Update: M{"$inc": M{"balance": 100}}, - }} err = s.runner.ResumeAll() c.Assert(err, IsNil) c.Logf("%8.3fs to ResumeAll N=%d 'prepared' txns", @@ -975,4 +969,3 @@ func (s *S) TestTxnQueuePreparing(c *C) { } c.Check(len(qdoc.Queue), Equals, expectedCount) } - From fd79249d77d3187875dbcc543ebcabf8c595e7c9 Mon Sep 17 00:00:00 2001 From: Dom Date: Mon, 9 Oct 2017 10:50:31 +0100 Subject: [PATCH 17/38] readme: credit @feliixx (#46) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 14db4305d..7b8ebb29b 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Further PR's (with tests) are welcome, but please maintain backwards compatibili * Support for [collations](https://docs.mongodb.com/manual/reference/collation/) in 3.4+ ([details](https://github.com/globalsign/mgo/pull/37)) * Provide BSON constants for convenience/sanity ([details](https://github.com/globalsign/mgo/pull/41)) * Consistently unmarshal time.Time values as UTC ([details](https://github.com/globalsign/mgo/pull/42)) - +* Enforces best practise coding guidelines ([details](https://github.com/globalsign/mgo/pull/44)) --- From dba7b4cdabd39ceb6a46d9a356312e063d42d023 Mon Sep 17 00:00:00 2001 From: "Artem V. Navrotskiy" Date: Wed, 11 Oct 2017 14:22:32 +0300 Subject: [PATCH 18/38] Fix GetBSON() method usage (#40) * Fix GetBSON() method usage Original issue --- You can't use type with custom GetBSON() method mixed with structure field type and structure field reference type. For example, you can't create custom GetBSON() for Bar type: ``` struct Foo { a Bar b *Bar } ``` Type implementation (`func (t Bar) GetBSON()` ) would crash on `Foo.b = nil` value encoding. Reference implementation (`func (t *Bar) GetBSON()` ) would not call on `Foo.a` value encoding. After this change --- For type implementation `func (t Bar) GetBSON()` would not call on `Foo.b = nil` value encoding. In this case `nil` value would be seariazied as `nil` BSON value. For reference implementation `func (t *Bar) GetBSON()` would call even on `Foo.a` value encoding. * Minor refactoring --- bson/bson_test.go | 65 +++++++++++++++++++++++++++++++++++++++++++++++ bson/decode.go | 24 +++++++++-------- bson/encode.go | 64 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 141 insertions(+), 12 deletions(-) diff --git a/bson/bson_test.go b/bson/bson_test.go index 6b8c0cd0b..695f9029d 100644 --- a/bson/bson_test.go +++ b/bson/bson_test.go @@ -36,6 +36,7 @@ import ( "reflect" "testing" "time" + "strings" "github.com/globalsign/mgo/bson" . "gopkg.in/check.v1" @@ -381,8 +382,54 @@ func (s *S) Test64bitInt(c *C) { // -------------------------------------------------------------------------- // Generic two-way struct marshaling tests. +type prefixPtr string +type prefixVal string + +func (t *prefixPtr) GetBSON() (interface{}, error) { + if t == nil { + return nil, nil + } + return "foo-" + string(*t), nil +} + +func (t *prefixPtr) SetBSON(raw bson.Raw) error { + var s string + if raw.Kind == 0x0A { + return bson.ErrSetZero + } + if err := raw.Unmarshal(&s); err != nil { + return err + } + if !strings.HasPrefix(s, "foo-") { + return errors.New("Prefix not found: " + s) + } + *t = prefixPtr(s[4:]) + return nil +} + +func (t prefixVal) GetBSON() (interface{}, error) { + return "foo-" + string(t), nil +} + +func (t *prefixVal) SetBSON(raw bson.Raw) error { + var s string + if raw.Kind == 0x0A { + return bson.ErrSetZero + } + if err := raw.Unmarshal(&s); err != nil { + return err + } + if !strings.HasPrefix(s, "foo-") { + return errors.New("Prefix not found: " + s) + } + *t = prefixVal(s[4:]) + return nil +} + var bytevar = byte(8) var byteptr = &bytevar +var prefixptr = prefixPtr("bar") +var prefixval = prefixVal("bar") var structItems = []testItemType{ {&struct{ Ptr *byte }{nil}, @@ -419,6 +466,24 @@ var structItems = []testItemType{ // Byte arrays. {&struct{ V [2]byte }{[2]byte{'y', 'o'}}, "\x05v\x00\x02\x00\x00\x00\x00yo"}, + + {&struct{ V prefixPtr }{prefixPtr("buzz")}, + "\x02v\x00\x09\x00\x00\x00foo-buzz\x00"}, + + {&struct{ V *prefixPtr }{&prefixptr}, + "\x02v\x00\x08\x00\x00\x00foo-bar\x00"}, + + {&struct{ V *prefixPtr }{nil}, + "\x0Av\x00"}, + + {&struct{ V prefixVal }{prefixVal("buzz")}, + "\x02v\x00\x09\x00\x00\x00foo-buzz\x00"}, + + {&struct{ V *prefixVal }{&prefixval}, + "\x02v\x00\x08\x00\x00\x00foo-bar\x00"}, + + {&struct{ V *prefixVal }{nil}, + "\x0Av\x00"}, } func (s *S) TestMarshalStructItems(c *C) { diff --git a/bson/decode.go b/bson/decode.go index 3b9e2856d..3e257f846 100644 --- a/bson/decode.go +++ b/bson/decode.go @@ -87,18 +87,20 @@ func setterStyle(outt reflect.Type) int { setterMutex.RLock() style := setterStyles[outt] setterMutex.RUnlock() - if style == setterUnknown { - setterMutex.Lock() - defer setterMutex.Unlock() - if outt.Implements(setterIface) { - setterStyles[outt] = setterType - } else if reflect.PtrTo(outt).Implements(setterIface) { - setterStyles[outt] = setterAddr - } else { - setterStyles[outt] = setterNone - } - style = setterStyles[outt] + if style != setterUnknown { + return style + } + + setterMutex.Lock() + defer setterMutex.Unlock() + if outt.Implements(setterIface) { + style = setterType + } else if reflect.PtrTo(outt).Implements(setterIface) { + style = setterAddr + } else { + style = setterNone } + setterStyles[outt] = style return style } diff --git a/bson/encode.go b/bson/encode.go index 75e503b57..61f388fa1 100644 --- a/bson/encode.go +++ b/bson/encode.go @@ -35,6 +35,7 @@ import ( "reflect" "sort" "strconv" + "sync" "time" ) @@ -60,13 +61,28 @@ var ( const itoaCacheSize = 32 +const ( + getterUnknown = iota + getterNone + getterTypeVal + getterTypePtr + getterAddr +) + var itoaCache []string +var getterStyles map[reflect.Type]int +var getterIface reflect.Type +var getterMutex sync.RWMutex + func init() { itoaCache = make([]string, itoaCacheSize) for i := 0; i != itoaCacheSize; i++ { itoaCache[i] = strconv.Itoa(i) } + var iface Getter + getterIface = reflect.TypeOf(&iface).Elem() + getterStyles = make(map[reflect.Type]int) } func itoa(i int) string { @@ -76,6 +92,52 @@ func itoa(i int) string { return strconv.Itoa(i) } +func getterStyle(outt reflect.Type) int { + getterMutex.RLock() + style := getterStyles[outt] + getterMutex.RUnlock() + if style != getterUnknown { + return style + } + + getterMutex.Lock() + defer getterMutex.Unlock() + if outt.Implements(getterIface) { + vt := outt + for vt.Kind() == reflect.Ptr { + vt = vt.Elem() + } + if vt.Implements(getterIface) { + style = getterTypeVal + } else { + style = getterTypePtr + } + } else if reflect.PtrTo(outt).Implements(getterIface) { + style = getterAddr + } else { + style = getterNone + } + getterStyles[outt] = style + return style +} + +func getGetter(outt reflect.Type, out reflect.Value) Getter { + style := getterStyle(outt) + if style == getterNone { + return nil + } + if style == getterAddr { + if !out.CanAddr() { + return nil + } + return out.Addr().Interface().(Getter) + } + if style == getterTypeVal && out.Kind() == reflect.Ptr && out.IsNil() { + return nil + } + return out.Interface().(Getter) +} + // -------------------------------------------------------------------------- // Marshaling of the document value itself. @@ -253,7 +315,7 @@ func (e *encoder) addElem(name string, v reflect.Value, minSize bool) { return } - if getter, ok := v.Interface().(Getter); ok { + if getter := getGetter(v.Type(), v); getter != nil { getv, err := getter.GetBSON() if err != nil { panic(err) From 12fb1c2cf3283718319e2af3e556a005573d2c4e Mon Sep 17 00:00:00 2001 From: Dom Date: Wed, 11 Oct 2017 12:46:18 +0100 Subject: [PATCH 19/38] readme: credit @bozaro (#47) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7b8ebb29b..36688b3d0 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ Further PR's (with tests) are welcome, but please maintain backwards compatibili * Provide BSON constants for convenience/sanity ([details](https://github.com/globalsign/mgo/pull/41)) * Consistently unmarshal time.Time values as UTC ([details](https://github.com/globalsign/mgo/pull/42)) * Enforces best practise coding guidelines ([details](https://github.com/globalsign/mgo/pull/44)) +* GetBSON correctly handles structs with both fields and pointers ([details](https://github.com/globalsign/mgo/pull/40)) --- From 199dc25378490aa711a031dfac6950e3a43ac723 Mon Sep 17 00:00:00 2001 From: "Artem V. Navrotskiy" Date: Thu, 19 Oct 2017 14:49:23 +0300 Subject: [PATCH 20/38] Improve cursorData struct unmarshaling speed (#49) This change remove full BSON decoding on: - parsing to `bson.Raw` and `bson.DocElem` fields; - skipping unused BSON fields. --- bson/bson_test.go | 13 ++ bson/decode.go | 475 +++++++++++++++++++++++++++------------------- bson/encode.go | 1 + 3 files changed, 292 insertions(+), 197 deletions(-) diff --git a/bson/bson_test.go b/bson/bson_test.go index 695f9029d..db72d8a06 100644 --- a/bson/bson_test.go +++ b/bson/bson_test.go @@ -82,6 +82,19 @@ func testUnmarshal(c *C, data string, obj interface{}) { err := bson.Unmarshal([]byte(data), zero) c.Assert(err, IsNil) c.Assert(zero, DeepEquals, obj) + + testUnmarshalRawElements(c, []byte(data)) +} + +func testUnmarshalRawElements(c *C, data []byte) { + elems := []bson.RawDocElem{} + err := bson.Unmarshal(data, &elems) + c.Assert(err, IsNil) + for _, elem := range elems { + if elem.Value.Kind == bson.ElementDocument || elem.Value.Kind == bson.ElementArray { + testUnmarshalRawElements(c, elem.Value.Data) + } + } } type testItemType struct { diff --git a/bson/decode.go b/bson/decode.go index 3e257f846..e71eac23f 100644 --- a/bson/decode.go +++ b/bson/decode.go @@ -28,7 +28,9 @@ package bson import ( + "errors" "fmt" + "io" "math" "net/url" "reflect" @@ -56,13 +58,6 @@ func corrupted() { panic("Document is corrupted") } -func settableValueOf(i interface{}) reflect.Value { - v := reflect.ValueOf(i) - sv := reflect.New(v.Type()).Elem() - sv.Set(v) - return sv -} - // -------------------------------------------------------------------------- // Unmarshaling of documents. @@ -137,8 +132,7 @@ func (d *decoder) readDocTo(out reflect.Value) { out.Set(reflect.New(outt.Elem())) } if setter := getSetter(outt, out); setter != nil { - var raw Raw - d.readDocTo(reflect.ValueOf(&raw)) + raw := d.readRaw(ElementDocument) err := setter.SetBSON(raw) if _, ok := err.(*TypeError); err != nil && !ok { panic(err) @@ -156,7 +150,10 @@ func (d *decoder) readDocTo(out reflect.Value) { var fieldsMap map[string]fieldInfo var inlineMap reflect.Value - start := d.i + if outt == typeRaw { + out.Set(reflect.ValueOf(d.readRaw(ElementDocument))) + return + } origout := out if outk == reflect.Interface { @@ -195,22 +192,20 @@ func (d *decoder) readDocTo(out reflect.Value) { clearMap(out) } case reflect.Struct: - if outt != typeRaw { - sinfo, err := getStructInfo(out.Type()) - if err != nil { - panic(err) + sinfo, err := getStructInfo(out.Type()) + if err != nil { + panic(err) + } + fieldsMap = sinfo.FieldsMap + out.Set(sinfo.Zero) + if sinfo.InlineMap != -1 { + inlineMap = out.Field(sinfo.InlineMap) + if !inlineMap.IsNil() && inlineMap.Len() > 0 { + clearMap(inlineMap) } - fieldsMap = sinfo.FieldsMap - out.Set(sinfo.Zero) - if sinfo.InlineMap != -1 { - inlineMap = out.Field(sinfo.InlineMap) - if !inlineMap.IsNil() && inlineMap.Len() > 0 { - clearMap(inlineMap) - } - elemType = inlineMap.Type().Elem() - if elemType == typeIface { - d.docType = inlineMap.Type() - } + elemType = inlineMap.Type().Elem() + if elemType == typeIface { + d.docType = inlineMap.Type() } } case reflect.Slice: @@ -227,70 +222,58 @@ func (d *decoder) readDocTo(out reflect.Value) { panic("Unsupported document type for unmarshalling: " + out.Type().String()) } - if outt == typeRaw { - d.skipDoc() - } else { - end := int(d.readInt32()) - end += d.i - 4 - if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' { + end := int(d.readInt32()) + end += d.i - 4 + if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' { + corrupted() + } + for d.in[d.i] != '\x00' { + kind := d.readByte() + name := d.readCStr() + if d.i >= end { corrupted() } - for d.in[d.i] != '\x00' { - kind := d.readByte() - name := d.readCStr() - if d.i >= end { - corrupted() - } - switch outk { - case reflect.Map: - e := reflect.New(elemType).Elem() - if d.readElemTo(e, kind) { - k := reflect.ValueOf(name) - if convertKey { - k = k.Convert(keyType) - } - out.SetMapIndex(k, e) + switch outk { + case reflect.Map: + e := reflect.New(elemType).Elem() + if d.readElemTo(e, kind) { + k := reflect.ValueOf(name) + if convertKey { + k = k.Convert(keyType) } - case reflect.Struct: - if outt == typeRaw { - d.dropElem(kind) + out.SetMapIndex(k, e) + } + case reflect.Struct: + if info, ok := fieldsMap[name]; ok { + if info.Inline == nil { + d.readElemTo(out.Field(info.Num), kind) } else { - if info, ok := fieldsMap[name]; ok { - if info.Inline == nil { - d.readElemTo(out.Field(info.Num), kind) - } else { - d.readElemTo(out.FieldByIndex(info.Inline), kind) - } - } else if inlineMap.IsValid() { - if inlineMap.IsNil() { - inlineMap.Set(reflect.MakeMap(inlineMap.Type())) - } - e := reflect.New(elemType).Elem() - if d.readElemTo(e, kind) { - inlineMap.SetMapIndex(reflect.ValueOf(name), e) - } - } else { - d.dropElem(kind) - } + d.readElemTo(out.FieldByIndex(info.Inline), kind) } - case reflect.Slice: - } - - if d.i >= end { - corrupted() + } else if inlineMap.IsValid() { + if inlineMap.IsNil() { + inlineMap.Set(reflect.MakeMap(inlineMap.Type())) + } + e := reflect.New(elemType).Elem() + if d.readElemTo(e, kind) { + inlineMap.SetMapIndex(reflect.ValueOf(name), e) + } + } else { + d.dropElem(kind) } + case reflect.Slice: } - d.i++ // '\x00' - if d.i != end { + + if d.i >= end { corrupted() } } - d.docType = docType - - if outt == typeRaw { - out.Set(reflect.ValueOf(Raw{0x03, d.in[start:d.i]})) + d.i++ // '\x00' + if d.i != end { + corrupted() } + d.docType = docType } func (d *decoder) readArrayDocTo(out reflect.Value) { @@ -332,9 +315,12 @@ func (d *decoder) readSliceDoc(t reflect.Type) interface{} { tmp := make([]reflect.Value, 0, 8) elemType := t.Elem() if elemType == typeRawDocElem { - d.dropElem(0x04) + d.dropElem(ElementArray) return reflect.Zero(t).Interface() } + if elemType == typeRaw { + return d.readSliceOfRaw() + } end := int(d.readInt32()) end += d.i - 4 @@ -371,6 +357,151 @@ func (d *decoder) readSliceDoc(t reflect.Type) interface{} { return slice.Interface() } +func BSONElementSize(kind byte, offset int, buffer []byte) (int, error) { + switch kind { + case ElementFloat64: // Float64 + return 8, nil + case ElementJavaScriptWithoutScope: // JavaScript without scope + fallthrough + case ElementSymbol: // Symbol + fallthrough + case ElementString: // UTF-8 string + size, err := getSize(offset, buffer) + if err != nil { + return 0, err + } + if size < 1 { + return 0, errors.New("String size can't be less then one byte") + } + size += 4 + if offset+size > len(buffer) { + return 0, io.ErrUnexpectedEOF + } + if buffer[offset+size-1] != 0 { + return 0, errors.New("Invalid string: non zero-terminated") + } + return size, nil + case ElementArray: // Array + fallthrough + case ElementDocument: // Document + size, err := getSize(offset, buffer) + if err != nil { + return 0, err + } + if size < 5 { + return 0, errors.New("Declared document size is too small") + } + return size, nil + case ElementBinary: // Binary + size, err := getSize(offset, buffer) + if err != nil { + return 0, err + } + if size < 0 { + return 0, errors.New("Binary data size can't be negative") + } + return size + 5, nil + case Element06: // Undefined (obsolete, but still seen in the wild) + return 0, nil + case ElementObjectId: // ObjectId + return 12, nil + case ElementBool: // Bool + return 1, nil + case ElementDatetime: // Timestamp + return 8, nil + case ElementNil: // Nil + return 0, nil + case ElementRegEx: // RegEx + end := offset + for i := 0; i < 2; i++ { + for end < len(buffer) && buffer[end] != '\x00' { + end++ + } + end++ + } + if end > len(buffer) { + return 0, io.ErrUnexpectedEOF + } + return end - offset, nil + case ElementDBPointer: // DBPointer + size, err := getSize(offset, buffer) + if err != nil { + return 0, err + } + if size < 1 { + return 0, errors.New("String size can't be less then one byte") + } + return size + 12 + 4, nil + case ElementJavaScriptWithScope: // JavaScript with scope + size, err := getSize(offset, buffer) + if err != nil { + return 0, err + } + if size < 4+5+5 { + return 0, errors.New("Declared document element is too small") + } + return size, nil + case ElementInt32: // Int32 + return 4, nil + case ElementTimestamp: // Mongo-specific timestamp + return 8, nil + case ElementInt64: // Int64 + return 8, nil + case ElementDecimal128: // Decimal128 + return 16, nil + case ElementMaxKey: // Max key + return 0, nil + case ElementMinKey: // Min key + return 0, nil + default: + return 0, errors.New(fmt.Sprintf("Unknown element kind (0x%02X)", kind)) + } +} + +func (d *decoder) readRaw(kind byte) Raw { + size, err := BSONElementSize(kind, d.i, d.in) + if err != nil { + corrupted() + } + if d.i+size > len(d.in) { + corrupted() + } + d.i += size + return Raw{ + Kind: kind, + Data: d.in[d.i-size : d.i], + } +} + +func (d *decoder) readSliceOfRaw() interface{} { + tmp := make([]Raw, 0, 8) + end := int(d.readInt32()) + end += d.i - 4 + if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' { + corrupted() + } + for d.in[d.i] != '\x00' { + kind := d.readByte() + for d.i < end && d.in[d.i] != '\x00' { + d.i++ + } + if d.i >= end { + corrupted() + } + d.i++ + e := d.readRaw(kind) + tmp = append(tmp, e) + if d.i >= end { + corrupted() + } + } + d.i++ // '\x00' + if d.i != end { + corrupted() + } + return tmp +} + var typeSlice = reflect.TypeOf([]interface{}{}) var typeIface = typeSlice.Elem() @@ -396,11 +527,8 @@ func (d *decoder) readRawDocElems(typ reflect.Type) reflect.Value { d.docType = typ slice := make([]RawDocElem, 0, 8) d.readDocWith(func(kind byte, name string) { - e := RawDocElem{Name: name} - v := reflect.ValueOf(&e.Value) - if d.readElemTo(v.Elem(), kind) { - slice = append(slice, e) - } + e := RawDocElem{Name: name, Value: d.readRaw(kind)} + slice = append(slice, e) }) slicev := reflect.New(typ).Elem() slicev.Set(reflect.ValueOf(slice)) @@ -433,76 +561,35 @@ func (d *decoder) readDocWith(f func(kind byte, name string)) { // -------------------------------------------------------------------------- // Unmarshaling of individual elements within a document. - -var blackHole = settableValueOf(struct{}{}) - func (d *decoder) dropElem(kind byte) { - switch kind { - case 0x01, 0x09, 0x11, 0x12: // double, utc datetime, timestamp, int64 - d.i += 8 - case 0x02, 0x0D, 0x0E: // string, javascript, symbol - l := int(d.readInt32()) - if l <= 0 || d.i+l >= len(d.in) || d.in[d.i+l-1] != 0x00 { - corrupted() - } - d.i += l - case 0x03, 0x04: // doc, array - d.skipDoc() - case 0x05: // binary - l := int(d.readInt32()) - k := d.readByte() - if k == 0x02 && l > 4 { - rl := int(d.readInt32()) - if rl != l-4 { - corrupted() - } - } - d.i += l - case 0x06: // undefined - case 0x07: // objectId - d.i += 12 - case 0x08: - k := d.readByte() - if k != 0x00 && k != 0x01 { - corrupted() - } - case 0x0A: // null - case 0x0B: // regex - d.readCStr() - d.readCStr() - case 0x0C: // dbpointer - d.dropElem(0x02) - d.i += 12 - case 0x0F: - start := d.i - l := int(d.readInt32()) - d.dropElem(0x02) // string - d.skipDoc() - if d.i != start+l { - corrupted() - } - case 0x10: // int32 - d.i += 4 - case 0x13: // decimal - d.i += 16 - case 0xFF, 0x7F: //min key, max key - default: - d.readElemTo(blackHole, kind) + size, err := BSONElementSize(kind, d.i, d.in) + if err != nil { + corrupted() } - - if d.i > len(d.in) { + if d.i+size > len(d.in) { corrupted() } + d.i += size } // Attempt to decode an element from the document and put it into out. // If the types are not compatible, the returned ok value will be // false and out will be unchanged. func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { + outt := out.Type() - start := d.i + if outt == typeRaw { + out.Set(reflect.ValueOf(d.readRaw(kind))) + return true + } - if kind == 0x03 { + if outt == typeRawPtr { + raw := d.readRaw(kind) + out.Set(reflect.ValueOf(&raw)) + return true + } + + if kind == ElementDocument { // Delegate unmarshaling of documents. outt := out.Type() outk := out.Kind() @@ -522,24 +609,39 @@ func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { case typeRawDocElem: out.Set(d.readRawDocElems(outt)) default: - d.skipDoc() + d.dropElem(kind) } return true } - d.skipDoc() + d.dropElem(kind) return true } + if setter := getSetter(outt, out); setter != nil { + err := setter.SetBSON(d.readRaw(kind)) + if err == ErrSetZero { + out.Set(reflect.Zero(outt)) + return true + } + if err == nil { + return true + } + if _, ok := err.(*TypeError); !ok { + panic(err) + } + return false + } + var in interface{} switch kind { - case 0x01: // Float64 + case ElementFloat64: in = d.readFloat64() - case 0x02: // UTF-8 string + case ElementString: in = d.readStr() - case 0x03: // Document + case ElementDocument: panic("Can't happen. Handled above.") - case 0x04: // Array + case ElementArray: outt := out.Type() if setterStyle(outt) != setterNone { // Skip the value so its data is handed to the setter below. @@ -558,20 +660,20 @@ func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { default: in = d.readSliceDoc(typeSlice) } - case 0x05: // Binary + case ElementBinary: b := d.readBinary() - if b.Kind == 0x00 || b.Kind == 0x02 { + if b.Kind == BinaryGeneric || b.Kind == BinaryBinaryOld { in = b.Data } else { in = b } - case 0x06: // Undefined (obsolete, but still seen in the wild) + case Element06: // Undefined (obsolete, but still seen in the wild) in = Undefined - case 0x07: // ObjectId + case ElementObjectId: in = ObjectId(d.readBytes(12)) - case 0x08: // Bool + case ElementBool: in = d.readBool() - case 0x09: // Timestamp + case ElementDatetime: // Timestamp // MongoDB handles timestamps as milliseconds. i := d.readInt64() if i == -62135596800000 { @@ -579,17 +681,17 @@ func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { } else { in = time.Unix(i/1e3, i%1e3*1e6).UTC() } - case 0x0A: // Nil + case ElementNil: in = nil - case 0x0B: // RegEx + case ElementRegEx: in = d.readRegEx() - case 0x0C: + case ElementDBPointer: in = DBPointer{Namespace: d.readStr(), Id: ObjectId(d.readBytes(12))} - case 0x0D: // JavaScript without scope + case ElementJavaScriptWithoutScope: in = JavaScript{Code: d.readStr()} - case 0x0E: // Symbol + case ElementSymbol: in = Symbol(d.readStr()) - case 0x0F: // JavaScript with scope + case ElementJavaScriptWithScope: start := d.i l := int(d.readInt32()) js := JavaScript{d.readStr(), make(M)} @@ -598,52 +700,30 @@ func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { corrupted() } in = js - case 0x10: // Int32 + case ElementInt32: in = int(d.readInt32()) - case 0x11: // Mongo-specific timestamp + case ElementTimestamp: // Mongo-specific timestamp in = MongoTimestamp(d.readInt64()) - case 0x12: // Int64 + case ElementInt64: switch out.Type() { case typeTimeDuration: in = time.Duration(time.Duration(d.readInt64()) * time.Millisecond) default: in = d.readInt64() } - case 0x13: // Decimal128 + case ElementDecimal128: in = Decimal128{ l: uint64(d.readInt64()), h: uint64(d.readInt64()), } - case 0x7F: // Max key + case ElementMaxKey: in = MaxKey - case 0xFF: // Min key + case ElementMinKey: in = MinKey default: panic(fmt.Sprintf("Unknown element kind (0x%02X)", kind)) } - outt := out.Type() - - if outt == typeRaw { - out.Set(reflect.ValueOf(Raw{kind, d.in[start:d.i]})) - return true - } - - if setter := getSetter(outt, out); setter != nil { - err := setter.SetBSON(Raw{kind, d.in[start:d.i]}) - if err == ErrSetZero { - out.Set(reflect.Zero(outt)) - return true - } - if err == nil { - return true - } - if _, ok := err.(*TypeError); !ok { - panic(err) - } - return false - } - if in == nil { out.Set(reflect.Zero(outt)) return true @@ -818,15 +898,6 @@ func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { // -------------------------------------------------------------------------- // Parsers of basic types. -func (d *decoder) skipDoc() { - end := int(d.readInt32()) - end += d.i - 4 - if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' { - corrupted() - } - d.i = end -} - func (d *decoder) readRegEx() RegEx { re := RegEx{} re.Pattern = d.readCStr() @@ -838,7 +909,7 @@ func (d *decoder) readBinary() Binary { l := d.readInt32() b := Binary{} b.Kind = d.readByte() - if b.Kind == 0x02 && l > 4 { + if b.Kind == BinaryBinaryOld && l > 4 { // Weird obsolete format with redundant length. rl := d.readInt32() if rl != l-4 { @@ -898,6 +969,16 @@ func (d *decoder) readInt32() int32 { (uint32(b[3]) << 24)) } +func getSize(offset int, b []byte) (int, error) { + if offset+4 > len(b) { + return 0, io.ErrUnexpectedEOF + } + return int((uint32(b[offset]) << 0) | + (uint32(b[offset+1]) << 8) | + (uint32(b[offset+2]) << 16) | + (uint32(b[offset+3]) << 24)), nil +} + func (d *decoder) readInt64() int64 { b := d.readBytes(8) return int64((uint64(b[0]) << 0) | diff --git a/bson/encode.go b/bson/encode.go index 61f388fa1..f307c31ec 100644 --- a/bson/encode.go +++ b/bson/encode.go @@ -52,6 +52,7 @@ var ( typeDocElem = reflect.TypeOf(DocElem{}) typeRawDocElem = reflect.TypeOf(RawDocElem{}) typeRaw = reflect.TypeOf(Raw{}) + typeRawPtr = reflect.PtrTo(reflect.TypeOf(Raw{})) typeURL = reflect.TypeOf(url.URL{}) typeTime = reflect.TypeOf(time.Time{}) typeString = reflect.TypeOf("") From 345ab0b32cfcf5557098c832c02e3fe3b4536a1b Mon Sep 17 00:00:00 2001 From: Dom Date: Thu, 19 Oct 2017 15:24:18 +0100 Subject: [PATCH 21/38] readme: credit @bozaro and @idy (#53) * readme: credit @bozaro and @idy * readme: add @idy to contributor list --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 36688b3d0..87cde972e 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ Further PR's (with tests) are welcome, but please maintain backwards compatibili * Consistently unmarshal time.Time values as UTC ([details](https://github.com/globalsign/mgo/pull/42)) * Enforces best practise coding guidelines ([details](https://github.com/globalsign/mgo/pull/44)) * GetBSON correctly handles structs with both fields and pointers ([details](https://github.com/globalsign/mgo/pull/40)) +* Improved bson.Raw unmarshalling performance ([details](https://github.com/globalsign/mgo/pull/49)) +* Minimise socket connection timeouts due to excessive locking ([details](https://github.com/globalsign/mgo/pull/52)) --- @@ -42,6 +44,7 @@ Further PR's (with tests) are welcome, but please maintain backwards compatibili * @eaglerayp * @feliixx * @fmpwizard +* @idy * @jameinel * @gazoon * @mapete94 From 0454966c021aa1737c9680aceef7e2ba26952fe3 Mon Sep 17 00:00:00 2001 From: Dom Date: Thu, 19 Oct 2017 15:27:19 +0100 Subject: [PATCH 22/38] do not lock while writing to a socket (#52) (#54) fix #51 --- socket.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/socket.go b/socket.go index 72fab9cf7..f6158189c 100644 --- a/socket.go +++ b/socket.go @@ -549,16 +549,15 @@ func (socket *mongoSocket) Query(ops ...interface{}) (err error) { socket.replyFuncs[requestId] = request.replyFunc requestId++ } - + socket.Unlock() debugf("Socket %p to %s: sending %d op(s) (%d bytes)", socket, socket.addr, len(ops), len(buf)) - stats.sentOps(len(ops)) + stats.sentOps(len(ops)) socket.updateDeadline(writeDeadline) _, err = socket.conn.Write(buf) if !wasWaiting && requestCount > 0 { socket.updateDeadline(readDeadline) } - socket.Unlock() return err } From 663dfe518a3aebd12170b48170060fb0f91d3627 Mon Sep 17 00:00:00 2001 From: Ceysun Sucu Date: Thu, 2 Nov 2017 14:52:41 +0000 Subject: [PATCH 23/38] Add proper DN construction (#55) * Add proper DN construction * Added openssl check to test * Addressed code review comments * Changes to RDNformatting, and test * type/value fields to tests * Changes to RDN formatting * Corrected comment in getRFC2253NameString * Corrected comment in getRFC2253NameString * Changes to login and rdn formatting * Changed escaping of # --- auth_test.go | 52 ++++++++++++++++++++++ session.go | 95 ++++++++++++++++++++++++++++++++++++++++ session_internal_test.go | 44 ++++++++++++++++++- 3 files changed, 189 insertions(+), 2 deletions(-) diff --git a/auth_test.go b/auth_test.go index ed1af5abf..f9abbdd61 100644 --- a/auth_test.go +++ b/auth_test.go @@ -28,6 +28,7 @@ package mgo_test import ( "crypto/tls" + "crypto/x509" "flag" "fmt" "io/ioutil" @@ -963,6 +964,57 @@ func (s *S) TestAuthX509Cred(c *C) { c.Assert(len(names) > 0, Equals, true) } +func (s *S) TestAuthX509CredRDNConstruction(c *C) { + session, err := mgo.Dial("localhost:40001") + c.Assert(err, IsNil) + defer session.Close() + binfo, err := session.BuildInfo() + c.Assert(err, IsNil) + if binfo.OpenSSLVersion == "" { + c.Skip("server does not support SSL") + } + + clientCertPEM, err := ioutil.ReadFile("harness/certs/client.pem") + c.Assert(err, IsNil) + + clientCert, err := tls.X509KeyPair(clientCertPEM, clientCertPEM) + c.Assert(err, IsNil) + + clientCert.Leaf, err = x509.ParseCertificate(clientCert.Certificate[0]) + c.Assert(err, IsNil) + + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + Certificates: []tls.Certificate{clientCert}, + } + + var host = "localhost:40003" + c.Logf("Connecting to %s...", host) + session, err = mgo.DialWithInfo(&mgo.DialInfo{ + Addrs: []string{host}, + DialServer: func(addr *mgo.ServerAddr) (net.Conn, error) { + return tls.Dial("tcp", addr.String(), tlsConfig) + }, + }) + c.Assert(err, IsNil) + defer session.Close() + + cred := &mgo.Credential{ + Username: "root", + Mechanism: "MONGODB-X509", + Source: "$external", + Certificate: tlsConfig.Certificates[0].Leaf, + } + err = session.Login(cred) + c.Assert(err, NotNil) + + cred.Username = "" + c.Logf("Authenticating...") + err = session.Login(cred) + c.Assert(err, IsNil) + c.Logf("Authenticated!") +} + var ( plainFlag = flag.String("plain", "", "Host to test PLAIN authentication against (depends on custom environment)") plainUser = "einstein" diff --git a/session.go b/session.go index 074f48688..91adfab67 100644 --- a/session.go +++ b/session.go @@ -28,6 +28,9 @@ package mgo import ( "crypto/md5" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" "encoding/hex" "errors" "fmt" @@ -825,6 +828,15 @@ type Credential struct { // Mechanism defines the protocol for credential negotiation. // Defaults to "MONGODB-CR". Mechanism string + + // Certificate defines an x509 certificate for authentication at login, + // for reference please see, https://docs.mongodb.com/manual/tutorial/configure-x509-client-authentication/ + // If providing a certificate: + // The Username field is populated from the cert and should not be set + // The Mechanism field should be MONGODB-X509 or not set. + // The Source field should be $external or not set. + // If not specified, the username will have to be set manually. + Certificate *x509.Certificate } // Login authenticates with MongoDB using the provided credential. The @@ -847,6 +859,19 @@ func (s *Session) Login(cred *Credential) error { defer socket.Release() credCopy := *cred + if cred.Certificate != nil && cred.Username != "" { + return errors.New("failed to login, both certificate and credentials are given") + } + + if cred.Certificate != nil { + credCopy.Username, err = getRFC2253NameStringFromCert(cred.Certificate) + if err != nil { + return err + } + credCopy.Mechanism = "MONGODB-X509" + credCopy.Source = "$external" + } + if cred.Source == "" { if cred.Mechanism == "GSSAPI" { credCopy.Source = "$external" @@ -5212,3 +5237,73 @@ func hasErrMsg(d []byte) bool { } return false } + +// getRFC2253NameStringFromCert converts from an ASN.1 structured representation of the certificate +// to a UTF-8 string representation(RDN) and returns it. +func getRFC2253NameStringFromCert(certificate *x509.Certificate) (string, error) { + var RDNElements = pkix.RDNSequence{} + _, err := asn1.Unmarshal(certificate.RawSubject, &RDNElements) + return getRFC2253NameString(&RDNElements), err +} + +// getRFC2253NameString converts from an ASN.1 structured representation of the RDNSequence +// from the certificate to a UTF-8 string representation(RDN) and returns it. +func getRFC2253NameString(RDNElements *pkix.RDNSequence) string { + var RDNElementsString = []string{} + var replacer = strings.NewReplacer(",", "\\,", "=", "\\=", "+", "\\+", "<", "\\<", ">", "\\>", ";", "\\;") + //The elements in the sequence needs to be reversed when converting them + for i := len(*RDNElements) - 1; i >= 0; i-- { + var nameAndValueList = make([]string,len((*RDNElements)[i])) + for j, attribute := range (*RDNElements)[i] { + var shortAttributeName = rdnOIDToShortName(attribute.Type) + if len(shortAttributeName) <= 0 { + nameAndValueList[j] = fmt.Sprintf("%s=%X", attribute.Type.String(), attribute.Value.([]byte)) + continue + } + var attributeValueString = attribute.Value.(string) + // escape leading space or # + if strings.HasPrefix(attributeValueString, " ") || strings.HasPrefix(attributeValueString, "#") { + attributeValueString = "\\" + attributeValueString + } + // escape trailing space, unless it's already escaped + if strings.HasSuffix(attributeValueString, " ") && !strings.HasSuffix(attributeValueString, "\\ ") { + attributeValueString = attributeValueString[:len(attributeValueString)-1] + "\\ " + } + + // escape , = + < > # ; + attributeValueString = replacer.Replace(attributeValueString) + nameAndValueList[j] = fmt.Sprintf("%s=%s", shortAttributeName, attributeValueString) + } + + RDNElementsString = append(RDNElementsString, strings.Join(nameAndValueList, "+")) + } + + return strings.Join(RDNElementsString, ",") +} + +var oidsToShortNames = []struct { + oid asn1.ObjectIdentifier + shortName string +}{ + {asn1.ObjectIdentifier{2, 5, 4, 3}, "CN"}, + {asn1.ObjectIdentifier{2, 5, 4, 6}, "C"}, + {asn1.ObjectIdentifier{2, 5, 4, 7}, "L"}, + {asn1.ObjectIdentifier{2, 5, 4, 8}, "ST"}, + {asn1.ObjectIdentifier{2, 5, 4, 10}, "O"}, + {asn1.ObjectIdentifier{2, 5, 4, 11}, "OU"}, + {asn1.ObjectIdentifier{2, 5, 4, 9}, "STREET"}, + {asn1.ObjectIdentifier{0, 9, 2342, 19200300, 100, 1, 25}, "DC"}, + {asn1.ObjectIdentifier{0, 9, 2342, 19200300, 100, 1, 1}, "UID"}, +} + +// rdnOIDToShortName returns an short name of the given RDN OID. If the OID does not have a short +// name, the function returns an empty string +func rdnOIDToShortName(oid asn1.ObjectIdentifier) string { + for i := range oidsToShortNames { + if oidsToShortNames[i].oid.Equal(oid) { + return oidsToShortNames[i].shortName + } + } + + return "" +} diff --git a/session_internal_test.go b/session_internal_test.go index f5f796c99..ddce59cae 100644 --- a/session_internal_test.go +++ b/session_internal_test.go @@ -1,11 +1,17 @@ package mgo import ( - "testing" - + "crypto/x509/pkix" + "encoding/asn1" "github.com/globalsign/mgo/bson" + . "gopkg.in/check.v1" + "testing" ) +type S struct{} + +var _ = Suite(&S{}) + // This file is for testing functions that are not exported outside the mgo // package - avoid doing so if at all possible. @@ -22,3 +28,37 @@ func TestIndexedInt64FieldsBug(t *testing.T) { _ = simpleIndexKey(input) } + +func (s *S) TestGetRFC2253NameStringSingleValued(c *C) { + var RDNElements = pkix.RDNSequence{ + {{Type: asn1.ObjectIdentifier{2, 5, 4, 6}, Value: "GO"}}, + {{Type: asn1.ObjectIdentifier{2, 5, 4, 8}, Value: "MGO"}}, + {{Type: asn1.ObjectIdentifier{2, 5, 4, 7}, Value: "MGO"}}, + {{Type: asn1.ObjectIdentifier{2, 5, 4, 10}, Value: "MGO"}}, + {{Type: asn1.ObjectIdentifier{2, 5, 4, 11}, Value: "Client"}}, + {{Type: asn1.ObjectIdentifier{2, 5, 4, 3}, Value: "localhost"}}, + } + + c.Assert(getRFC2253NameString(&RDNElements), Equals, "CN=localhost,OU=Client,O=MGO,L=MGO,ST=MGO,C=GO") +} + +func (s *S) TestGetRFC2253NameStringEscapeChars(c *C) { + var RDNElements = pkix.RDNSequence{ + {{Type: asn1.ObjectIdentifier{2, 5, 4, 6}, Value: "GB"}}, + {{Type: asn1.ObjectIdentifier{2, 5, 4, 8}, Value: "MGO "}}, + {{Type: asn1.ObjectIdentifier{2, 5, 4, 10}, Value: "Sue, Grabbit and Runn < > ;"}}, + {{Type: asn1.ObjectIdentifier{2, 5, 4, 3}, Value: "L. Eagle"}}, + } + + c.Assert(getRFC2253NameString(&RDNElements), Equals, "CN=L. Eagle,O=Sue\\, Grabbit and Runn \\< \\> \\;,ST=MGO\\ ,C=GB") +} + +func (s *S) TestGetRFC2253NameStringMultiValued(c *C) { + var RDNElements = pkix.RDNSequence{ + {{Type: asn1.ObjectIdentifier{2, 5, 4, 6}, Value: "US"}}, + {{Type: asn1.ObjectIdentifier{2, 5, 4, 10}, Value: "Widget Inc."}}, + {{Type: asn1.ObjectIdentifier{2, 5, 4, 11}, Value: "Sales"}, {Type: asn1.ObjectIdentifier{2, 5, 4, 3}, Value: "J. Smith"}}, + } + + c.Assert(getRFC2253NameString(&RDNElements), Equals, "OU=Sales+CN=J. Smith,O=Widget Inc.,C=US") +} From 7cd0b89475dc8d0d92bb1a90288a14738c36c8e7 Mon Sep 17 00:00:00 2001 From: Adrien Petel Date: Fri, 3 Nov 2017 16:53:55 +0400 Subject: [PATCH 24/38] reduce memory allocation in bulk op (#56) use memory pooling to reuse bulkActions and avoid some allocations --- bulk.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/bulk.go b/bulk.go index d6925fba4..c234fccee 100644 --- a/bulk.go +++ b/bulk.go @@ -3,6 +3,7 @@ package mgo import ( "bytes" "sort" + "sync" "github.com/globalsign/mgo/bson" ) @@ -118,6 +119,15 @@ func (e *BulkError) Cases() []BulkErrorCase { return e.ecases } +var actionPool = sync.Pool{ + New: func() interface{} { + return &bulkAction{ + docs: make([]interface{}, 0), + idxs: make([]int, 0), + } + }, +} + // Bulk returns a value to prepare the execution of a bulk operation. func (c *Collection) Bulk() *Bulk { return &Bulk{c: c, ordered: true} @@ -145,7 +155,9 @@ func (b *Bulk) action(op bulkOp, opcount int) *bulkAction { } } if action == nil { - b.actions = append(b.actions, bulkAction{op: op}) + a := actionPool.Get().(*bulkAction) + a.op = op + b.actions = append(b.actions, *a) action = &b.actions[len(b.actions)-1] } for i := 0; i < opcount; i++ { @@ -288,6 +300,9 @@ func (b *Bulk) Run() (*BulkResult, error) { default: panic("unknown bulk operation") } + action.idxs = action.idxs[0:0] + action.docs = action.docs[0:0] + actionPool.Put(action) if !ok { failed = true if b.ordered { From ea8e8e616d6f8459f0bbdecff0b05a9540483b3a Mon Sep 17 00:00:00 2001 From: Dom Date: Mon, 6 Nov 2017 11:17:49 +0000 Subject: [PATCH 25/38] readme: credit @feliixx (#58) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 87cde972e..a2f608458 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Further PR's (with tests) are welcome, but please maintain backwards compatibili * Fixes attempting to authenticate before every query ([details](https://github.com/go-mgo/mgo/issues/254)) * Removes bulk update / delete batch size limitations ([details](https://github.com/go-mgo/mgo/issues/288)) * Adds native support for `time.Duration` marshalling ([details](https://github.com/go-mgo/mgo/pull/373)) -* Reduce memory footprint / garbage collection pressure by reusing buffers ([details](https://github.com/go-mgo/mgo/pull/229)) +* Reduce memory footprint / garbage collection pressure by reusing buffers ([details](https://github.com/go-mgo/mgo/pull/229), [more](https://github.com/globalsign/mgo/pull/56)) * Support majority read concerns ([details](https://github.com/globalsign/mgo/pull/2)) * Improved connection handling ([details](https://github.com/globalsign/mgo/pull/5)) * Hides SASL warnings ([details](https://github.com/globalsign/mgo/pull/7)) From 90c056c16eb749ea476643a0aa62d53035ac983e Mon Sep 17 00:00:00 2001 From: Adrien Petel Date: Tue, 12 Dec 2017 17:14:41 +0400 Subject: [PATCH 26/38] MongoDB 3.6: implement the new wire protocol (#61) * test against 3.6 * update go and mongodb version - use last minor version for each major serie of mongodb - use travis 'go' param instead of eval "$(gimme $GO)" as it fails to install correctly last minor version ( 1.8.x notation for example) * test fixes on 3.2.17 also re-enable TestFindIterSnapshot as it was fixed a long time ago * fix X509 test fix TestAuthX509CredRDNConstruction test: need to create an user with {"username": subject} before trying to login * Fix auth test on 3.6-rc3 Make sure that "rs3/127.0.0.1/40031" is elected at primary. Create user before running 'addShard' command as it requires root access Also add a retry mechanism to make sure that shard are correctly added cf https://docs.mongodb.com/manual/tutorial/deploy-shard-cluster/ * implement OP_MSG wire protocole require maxWireVersion >= 6 on server side, and `experimental=opmsg` in the connection string - get `MaxMessageSizeBytes` and `MaxWriteBatchSize` from handshake - allow unacknowledged writes with `moreToCome` flag - split bulk operations in batch of `maxWriteBatchSize` - add 'experimental' param in URL. To enable an experimental feature, add `experimental=featureName` in the connection URL flush logout directly Previously, `flushLogout()` was called at the beginning of each Query to the database. To avoid these useless calls, we flush logout directly when `Logout()` or `LogoutAll()` is called * re-enable TestViewWithCollation SERVER-31049 is fixed in 3.4.10, so re-enable it * refactor memory pooling use the same pool for send and received messages. Slices are returned to the pool without being resized. Default allocation size might need to be updated (currently 256, no benchmarks available yet) * update to 3.6.0 stable --- .travis.yml | 24 +- auth.go | 57 +++-- auth_test.go | 111 +++++---- bulk_test.go | 34 +-- cluster.go | 32 +-- cluster_test.go | 122 +++++----- gridfs_test.go | 24 +- harness/daemons/.env | 10 - harness/daemons/db2/run | 1 - harness/daemons/db3/run | 1 - harness/mongojs/dropall.js | 47 ++-- harness/mongojs/init.js | 147 +++++++----- server.go | 27 ++- session.go | 191 +++++++++++++++- session_test.go | 371 ++++++++++++++---------------- socket.go | 458 ++++++++++++++++++++++++++++--------- suite_test.go | 2 + 17 files changed, 1058 insertions(+), 601 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8d4428d1f..43cc9c2b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,27 +2,23 @@ language: go go_import_path: github.com/globalsign/mgo +go: + - 1.8.x + - 1.9.x + env: global: - BUCKET=https://s3.eu-west-2.amazonaws.com/globalsign-mgo + - FASTDL=https://fastdl.mongodb.org/linux matrix: - - GO=1.7 MONGODB=x86_64-2.6.11 - - GO=1.8.x MONGODB=x86_64-2.6.11 - - GO=1.7 MONGODB=x86_64-3.0.9 - - GO=1.8.x MONGODB=x86_64-3.0.9 - - GO=1.7 MONGODB=x86_64-3.2.3-nojournal - - GO=1.8.x MONGODB=x86_64-3.2.3-nojournal - - GO=1.7 MONGODB=x86_64-3.2.12 - - GO=1.8.x MONGODB=x86_64-3.2.12 - - GO=1.7 MONGODB=x86_64-3.2.16 - - GO=1.8.x MONGODB=x86_64-3.2.16 - - GO=1.7 MONGODB=x86_64-3.4.8 - - GO=1.8.x MONGODB=x86_64-3.4.8 + - MONGODB=x86_64-ubuntu1404-3.0.15 + - MONGODB=x86_64-ubuntu1404-3.2.17 + - MONGODB=x86_64-ubuntu1404-3.4.10 + - MONGODB=x86_64-ubuntu1404-3.6.0 install: - - eval "$(gimme $GO)" - - wget $BUCKET/mongodb-linux-$MONGODB.tgz + - wget $FASTDL/mongodb-linux-$MONGODB.tgz - tar xzvf mongodb-linux-$MONGODB.tgz - export PATH=$PWD/mongodb-linux-$MONGODB/bin:$PATH diff --git a/auth.go b/auth.go index 75d2ebc36..ed3baad15 100644 --- a/auth.go +++ b/auth.go @@ -170,12 +170,6 @@ func (socket *mongoSocket) Login(cred Credential) error { return nil } } - if socket.dropLogout(cred) { - debugf("Socket %p to %s: login: db=%q user=%q (cached)", socket, socket.addr, cred.Source, cred.Username) - socket.creds = append(socket.creds, cred) - socket.Unlock() - return nil - } socket.Unlock() debugf("Socket %p to %s: login: db=%q user=%q", socket, socket.addr, cred.Source, cred.Username) @@ -412,36 +406,50 @@ func (socket *mongoSocket) Logout(db string) { cred, found := socket.dropAuth(db) if found { debugf("Socket %p to %s: logout: db=%q (flagged)", socket, socket.addr, db) - socket.logout = append(socket.logout, cred) + socket.Unlock() + err := socket.flushLogout(cred) + if err != nil { + debugf("fail to logout for cred %v; error: %v", cred, err) + } + } else { + socket.Unlock() } - socket.Unlock() } func (socket *mongoSocket) LogoutAll() { socket.Lock() if l := len(socket.creds); l > 0 { - debugf("Socket %p to %s: logout all (flagged %d)", socket, socket.addr, l) - socket.logout = append(socket.logout, socket.creds...) + credCopy := make([]Credential, l) + copy(credCopy, socket.creds) socket.creds = socket.creds[0:0] + socket.Unlock() + debugf("Socket %p to %s: logout all (flagged %d)", socket, socket.addr, l) + err := socket.flushLogout(credCopy...) + if err != nil { + debugf("fail to logout for cred %v, error: %v", credCopy, err) + } + } else { + socket.Unlock() } - socket.Unlock() } -func (socket *mongoSocket) flushLogout() (ops []interface{}) { - socket.Lock() - if l := len(socket.logout); l > 0 { +func (socket *mongoSocket) flushLogout(cred ...Credential) error { + if l := len(cred); l > 0 { debugf("Socket %p to %s: logout all (flushing %d)", socket, socket.addr, l) + ops := make([]interface{}, l) for i := 0; i != l; i++ { op := queryOp{} op.query = &logoutCmd{1} - op.collection = socket.logout[i].Source + ".$cmd" + op.collection = cred[i].Source + ".$cmd" op.limit = -1 - ops = append(ops, &op) + ops[i] = &op + } + err := socket.Query(ops...) + if err != nil { + return fmt.Errorf("fail to logout: %v", err) } - socket.logout = socket.logout[0:0] } - socket.Unlock() - return + return nil } func (socket *mongoSocket) dropAuth(db string) (cred Credential, found bool) { @@ -454,14 +462,3 @@ func (socket *mongoSocket) dropAuth(db string) (cred Credential, found bool) { } return cred, false } - -func (socket *mongoSocket) dropLogout(cred Credential) (found bool) { - for i, sockCred := range socket.logout { - if sockCred == cred { - copy(socket.logout[i:], socket.logout[i+1:]) - socket.logout = socket.logout[:len(socket.logout)-1] - return true - } - } - return false -} diff --git a/auth_test.go b/auth_test.go index f9abbdd61..b7298786f 100644 --- a/auth_test.go +++ b/auth_test.go @@ -39,14 +39,14 @@ import ( "sync" "time" - mgo "github.com/globalsign/mgo" + "github.com/globalsign/mgo" . "gopkg.in/check.v1" ) func (s *S) TestAuthLoginDatabase(c *C) { // Test both with a normal database and with an authenticated shard. for _, addr := range []string{"localhost:40002", "localhost:40203"} { - session, err := mgo.Dial(addr) + session, err := mgo.Dial(addr + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -70,7 +70,7 @@ func (s *S) TestAuthLoginDatabase(c *C) { func (s *S) TestAuthLoginSession(c *C) { // Test both with a normal database and with an authenticated shard. for _, addr := range []string{"localhost:40002", "localhost:40203"} { - session, err := mgo.Dial(addr) + session, err := mgo.Dial(addr + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -123,7 +123,7 @@ func (s *S) TestAuthLoginLogout(c *C) { } func (s *S) TestAuthLoginLogoutAll(c *C) { - session, err := mgo.Dial("localhost:40002") + session, err := mgo.Dial("localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -171,7 +171,7 @@ func (s *S) TestAuthUpsertUser(c *C) { if !s.versionAtLeast(2, 4) { c.Skip("UpsertUser only works on 2.4+") } - session, err := mgo.Dial("localhost:40002") + session, err := mgo.Dial("localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -252,7 +252,7 @@ func (s *S) TestAuthUpsertUserOtherDBRoles(c *C) { if !s.versionAtLeast(2, 4) { c.Skip("UpsertUser only works on 2.4+") } - session, err := mgo.Dial("localhost:40002") + session, err := mgo.Dial("localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -285,7 +285,7 @@ func (s *S) TestAuthUpsertUserUpdates(c *C) { if !s.versionAtLeast(2, 4) { c.Skip("UpsertUser only works on 2.4+") } - session, err := mgo.Dial("localhost:40002") + session, err := mgo.Dial("localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -313,7 +313,7 @@ func (s *S) TestAuthUpsertUserUpdates(c *C) { c.Assert(err, IsNil) // Login with the new user. - usession, err := mgo.Dial("myruser:mynewpass@localhost:40002/mydb") + usession, err := mgo.Dial("myruser:mynewpass@localhost:40002/mydb" + expFeaturesString) c.Assert(err, IsNil) defer usession.Close() @@ -332,7 +332,7 @@ func (s *S) TestAuthUpsertUserUpdates(c *C) { c.Assert(err, IsNil) // Dial again to ensure the password hasn't changed. - usession, err = mgo.Dial("myruser:mynewpass@localhost:40002/mydb") + usession, err = mgo.Dial("myruser:mynewpass@localhost:40002/mydb" + expFeaturesString) c.Assert(err, IsNil) defer usession.Close() @@ -342,7 +342,7 @@ func (s *S) TestAuthUpsertUserUpdates(c *C) { } func (s *S) TestAuthAddUser(c *C) { - session, err := mgo.Dial("localhost:40002") + session, err := mgo.Dial("localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -373,7 +373,7 @@ func (s *S) TestAuthAddUser(c *C) { } func (s *S) TestAuthAddUserReplaces(c *C) { - session, err := mgo.Dial("localhost:40002") + session, err := mgo.Dial("localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -400,7 +400,7 @@ func (s *S) TestAuthAddUserReplaces(c *C) { } func (s *S) TestAuthRemoveUser(c *C) { - session, err := mgo.Dial("localhost:40002") + session, err := mgo.Dial("localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -421,7 +421,7 @@ func (s *S) TestAuthRemoveUser(c *C) { } func (s *S) TestAuthLoginTwiceDoesNothing(c *C) { - session, err := mgo.Dial("localhost:40002") + session, err := mgo.Dial("localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -439,7 +439,7 @@ func (s *S) TestAuthLoginTwiceDoesNothing(c *C) { } func (s *S) TestAuthLoginLogoutLoginDoesNothing(c *C) { - session, err := mgo.Dial("localhost:40002") + session, err := mgo.Dial("localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -449,16 +449,17 @@ func (s *S) TestAuthLoginLogoutLoginDoesNothing(c *C) { oldStats := mgo.GetStats() - admindb.Logout() - err = admindb.Login("root", "rapadura") + admindb.Logout() // 1 op + err = admindb.Login("root", "rapadura") // 3 op c.Assert(err, IsNil) newStats := mgo.GetStats() - c.Assert(newStats.SentOps, Equals, oldStats.SentOps) + // Logout is flush directly + c.Assert(newStats.SentOps, Equals, oldStats.SentOps+4) } func (s *S) TestAuthLoginSwitchUser(c *C) { - session, err := mgo.Dial("localhost:40002") + session, err := mgo.Dial("localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -485,7 +486,7 @@ func (s *S) TestAuthLoginSwitchUser(c *C) { } func (s *S) TestAuthLoginChangePassword(c *C) { - session, err := mgo.Dial("localhost:40002") + session, err := mgo.Dial("localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -514,7 +515,7 @@ func (s *S) TestAuthLoginChangePassword(c *C) { } func (s *S) TestAuthLoginCachingWithSessionRefresh(c *C) { - session, err := mgo.Dial("localhost:40002") + session, err := mgo.Dial("localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -530,7 +531,7 @@ func (s *S) TestAuthLoginCachingWithSessionRefresh(c *C) { } func (s *S) TestAuthLoginCachingWithSessionCopy(c *C) { - session, err := mgo.Dial("localhost:40002") + session, err := mgo.Dial("localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -547,7 +548,7 @@ func (s *S) TestAuthLoginCachingWithSessionCopy(c *C) { } func (s *S) TestAuthLoginCachingWithSessionClone(c *C) { - session, err := mgo.Dial("localhost:40002") + session, err := mgo.Dial("localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -564,7 +565,7 @@ func (s *S) TestAuthLoginCachingWithSessionClone(c *C) { } func (s *S) TestAuthLoginCachingWithNewSession(c *C) { - session, err := mgo.Dial("localhost:40002") + session, err := mgo.Dial("localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -584,7 +585,7 @@ func (s *S) TestAuthLoginCachingAcrossPool(c *C) { // Logins are cached even when the connection goes back // into the pool. - session, err := mgo.Dial("localhost:40002") + session, err := mgo.Dial("localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -617,9 +618,9 @@ func (s *S) TestAuthLoginCachingAcrossPool(c *C) { err = other.DB("mydb").Login("myuser", "mypass") c.Assert(err, IsNil) - // Both logins were cached, so no ops. + // No more caching, logout is flush directly newStats := mgo.GetStats() - c.Assert(newStats.SentOps, Equals, oldStats.SentOps) + c.Assert(newStats.SentOps, Equals, oldStats.SentOps+6) // And they actually worked. err = other.DB("mydb").C("mycoll").Insert(M{"n": 1}) @@ -635,7 +636,7 @@ func (s *S) TestAuthLoginCachingAcrossPoolWithLogout(c *C) { // Now verify that logouts are properly flushed if they // are not revalidated after leaving the pool. - session, err := mgo.Dial("localhost:40002") + session, err := mgo.Dial("localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -664,12 +665,12 @@ func (s *S) TestAuthLoginCachingAcrossPoolWithLogout(c *C) { oldStats := mgo.GetStats() - err = other.DB("mydb").Login("myuser", "mypass") + err = other.DB("mydb").Login("myuser", "mypass") // 3 op c.Assert(err, IsNil) - // Login was cached, so no ops. + // No more caching, logout is flush directly newStats := mgo.GetStats() - c.Assert(newStats.SentOps, Equals, oldStats.SentOps) + c.Assert(newStats.SentOps, Equals, oldStats.SentOps+3) // Can't write, since root has been implicitly logged out // when the collection went into the pool, and not revalidated. @@ -686,7 +687,7 @@ func (s *S) TestAuthLoginCachingAcrossPoolWithLogout(c *C) { func (s *S) TestAuthEventual(c *C) { // Eventual sessions don't keep sockets around, so they are // an interesting test case. - session, err := mgo.Dial("localhost:40002") + session, err := mgo.Dial("localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -722,7 +723,7 @@ func (s *S) TestAuthEventual(c *C) { } func (s *S) TestAuthURL(c *C) { - session, err := mgo.Dial("mongodb://root:rapadura@localhost:40002/") + session, err := mgo.Dial("mongodb://root:rapadura@localhost:40002/" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -731,7 +732,7 @@ func (s *S) TestAuthURL(c *C) { } func (s *S) TestAuthURLWrongCredentials(c *C) { - session, err := mgo.Dial("mongodb://root:wrong@localhost:40002/") + session, err := mgo.Dial("mongodb://root:wrong@localhost:40002/" + expFeaturesString) if session != nil { session.Close() } @@ -742,7 +743,7 @@ func (s *S) TestAuthURLWrongCredentials(c *C) { func (s *S) TestAuthURLWithNewSession(c *C) { // When authentication is in the URL, the new session will // actually carry it on as well, even if logged out explicitly. - session, err := mgo.Dial("mongodb://root:rapadura@localhost:40002/") + session, err := mgo.Dial("mongodb://root:rapadura@localhost:40002/" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -759,7 +760,7 @@ func (s *S) TestAuthURLWithNewSession(c *C) { } func (s *S) TestAuthURLWithDatabase(c *C) { - session, err := mgo.Dial("mongodb://root:rapadura@localhost:40002") + session, err := mgo.Dial("mongodb://root:rapadura@localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -771,9 +772,9 @@ func (s *S) TestAuthURLWithDatabase(c *C) { for i := 0; i < 2; i++ { var url string if i == 0 { - url = "mongodb://myruser:mypass@localhost:40002/mydb" + url = "mongodb://myruser:mypass@localhost:40002/mydb" + expFeaturesString } else { - url = "mongodb://myruser:mypass@localhost:40002/admin?authSource=mydb" + url = "mongodb://myruser:mypass@localhost:40002/admin?authSource=mydb" + "&" + string(expFeaturesString[1:]) } usession, err := mgo.Dial(url) c.Assert(err, IsNil) @@ -797,7 +798,7 @@ func (s *S) TestDefaultDatabase(c *C) { } for _, test := range tests { - session, err := mgo.Dial(test.url) + session, err := mgo.Dial(test.url + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -814,7 +815,7 @@ func (s *S) TestAuthDirect(c *C) { // Direct connections must work to the master and slaves. for _, port := range []string{"40031", "40032", "40033"} { url := fmt.Sprintf("mongodb://root:rapadura@localhost:%s/?connect=direct", port) - session, err := mgo.Dial(url) + session, err := mgo.Dial(url + "&" + string(expFeaturesString[1:])) c.Assert(err, IsNil) defer session.Close() @@ -830,7 +831,7 @@ func (s *S) TestAuthDirectWithLogin(c *C) { // Direct connections must work to the master and slaves. for _, port := range []string{"40031", "40032", "40033"} { url := fmt.Sprintf("mongodb://localhost:%s/?connect=direct", port) - session, err := mgo.Dial(url) + session, err := mgo.Dial(url + "&" + string(expFeaturesString[1:])) c.Assert(err, IsNil) defer session.Close() @@ -858,7 +859,7 @@ func (s *S) TestAuthScramSha1Cred(c *C) { } host := "localhost:40002" c.Logf("Connecting to %s...", host) - session, err := mgo.Dial(host) + session, err := mgo.Dial(host + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -884,7 +885,7 @@ func (s *S) TestAuthScramSha1URL(c *C) { } host := "localhost:40002" c.Logf("Connecting to %s...", host) - session, err := mgo.Dial(fmt.Sprintf("root:rapadura@%s?authMechanism=SCRAM-SHA-1", host)) + session, err := mgo.Dial(fmt.Sprintf("root:rapadura@%s?authMechanism=SCRAM-SHA-1&%s", host, string(expFeaturesString[1:]))) c.Assert(err, IsNil) defer session.Close() @@ -896,7 +897,7 @@ func (s *S) TestAuthScramSha1URL(c *C) { } func (s *S) TestAuthX509Cred(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() binfo, err := session.BuildInfo() @@ -921,6 +922,9 @@ func (s *S) TestAuthX509Cred(c *C) { c.Logf("Connecting to %s...", host) session, err = mgo.DialWithInfo(&mgo.DialInfo{ Addrs: []string{host}, + ExperimentalFeatures: map[string]bool{ + "opmsg": true, + }, DialServer: func(addr *mgo.ServerAddr) (net.Conn, error) { return tls.Dial("tcp", addr.String(), tlsConfig) }, @@ -965,7 +969,7 @@ func (s *S) TestAuthX509Cred(c *C) { } func (s *S) TestAuthX509CredRDNConstruction(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() binfo, err := session.BuildInfo() @@ -992,6 +996,9 @@ func (s *S) TestAuthX509CredRDNConstruction(c *C) { c.Logf("Connecting to %s...", host) session, err = mgo.DialWithInfo(&mgo.DialInfo{ Addrs: []string{host}, + ExperimentalFeatures: map[string]bool{ + "opmsg": true, + }, DialServer: func(addr *mgo.ServerAddr) (net.Conn, error) { return tls.Dial("tcp", addr.String(), tlsConfig) }, @@ -1008,6 +1015,22 @@ func (s *S) TestAuthX509CredRDNConstruction(c *C) { err = session.Login(cred) c.Assert(err, NotNil) + err = session.Login(&mgo.Credential{Username: "root", Password: "rapadura"}) + c.Assert(err, IsNil) + + // This needs to be kept in sync with client.pem + x509Subject := "CN=localhost,OU=Client,O=MGO,L=MGO,ST=MGO,C=GO" + + externalDB := session.DB("$external") + var x509User = mgo.User{ + Username: x509Subject, + OtherDBRoles: map[string][]mgo.Role{"admin": {mgo.RoleRoot}}, + } + err = externalDB.UpsertUser(&x509User) + c.Assert(err, IsNil) + + session.LogoutAll() + cred.Username = "" c.Logf("Authenticating...") err = session.Login(cred) diff --git a/bulk_test.go b/bulk_test.go index fa91dc44c..cc552c614 100644 --- a/bulk_test.go +++ b/bulk_test.go @@ -32,7 +32,7 @@ import ( ) func (s *S) TestBulkInsert(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -52,7 +52,7 @@ func (s *S) TestBulkInsert(c *C) { } func (s *S) TestBulkInsertError(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -73,7 +73,7 @@ func (s *S) TestBulkInsertError(c *C) { } func (s *S) TestBulkInsertErrorUnordered(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -100,7 +100,7 @@ func (s *S) TestBulkInsertErrorUnorderedSplitBatch(c *C) { // into the proper size and delivers them one by one. This test ensures that // the behavior of unordered (that is, continue on error) remains correct // when errors happen and there are batches left. - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -132,7 +132,7 @@ func (s *S) TestBulkInsertErrorUnorderedSplitBatch(c *C) { } func (s *S) TestBulkErrorString(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -180,7 +180,7 @@ func (s *S) TestBulkErrorCases_2_6(c *C) { if !s.versionAtLeast(2, 6) { c.Skip("2.4- has poor bulk reporting") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -219,7 +219,7 @@ func (s *S) TestBulkErrorCases_2_4(c *C) { if s.versionAtLeast(2, 6) { c.Skip("2.6+ has better reporting") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -255,7 +255,7 @@ func (s *S) TestBulkErrorCases_2_4(c *C) { } func (s *S) TestBulkErrorCasesOrdered(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -289,7 +289,7 @@ func (s *S) TestBulkErrorCasesOrdered(c *C) { } func (s *S) TestBulkUpdate(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -318,7 +318,7 @@ func (s *S) TestBulkUpdate(c *C) { } func (s *S) TestBulkUpdateOver1000(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -340,7 +340,7 @@ func (s *S) TestBulkUpdateOver1000(c *C) { } func (s *S) TestBulkUpdateError(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -367,7 +367,7 @@ func (s *S) TestBulkUpdateError(c *C) { } func (s *S) TestBulkUpdateErrorUnordered(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -395,7 +395,7 @@ func (s *S) TestBulkUpdateErrorUnordered(c *C) { } func (s *S) TestBulkUpdateAll(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -424,7 +424,7 @@ func (s *S) TestBulkUpdateAll(c *C) { } func (s *S) TestBulkMixedUnordered(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -454,7 +454,7 @@ func (s *S) TestBulkMixedUnordered(c *C) { } func (s *S) TestBulkUpsert(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -478,7 +478,7 @@ func (s *S) TestBulkUpsert(c *C) { } func (s *S) TestBulkRemove(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -502,7 +502,7 @@ func (s *S) TestBulkRemove(c *C) { } func (s *S) TestBulkRemoveAll(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() diff --git a/cluster.go b/cluster.go index 7fc639c24..262e7eafb 100644 --- a/cluster.go +++ b/cluster.go @@ -132,15 +132,17 @@ func (cluster *mongoCluster) removeServer(server *mongoServer) { } type isMasterResult struct { - IsMaster bool - Secondary bool - Primary string - Hosts []string - Passives []string - Tags bson.D - Msg string - SetName string `bson:"setName"` - MaxWireVersion int `bson:"maxWireVersion"` + IsMaster bool + Secondary bool + Primary string + Hosts []string + Passives []string + Tags bson.D + Msg string + SetName string `bson:"setName"` + MaxWireVersion int `bson:"maxWireVersion"` + MaxWriteBatchSize int `bson:"maxWriteBatchSize"` + MaxMessageSizeBytes int `bson:"maxMessageSizeBytes"` } func (cluster *mongoCluster) isMaster(socket *mongoSocket, result *isMasterResult) error { @@ -240,11 +242,13 @@ func (cluster *mongoCluster) syncServer(server *mongoServer) (info *mongoServerI } info = &mongoServerInfo{ - Master: result.IsMaster, - Mongos: result.Msg == "isdbgrid", - Tags: result.Tags, - SetName: result.SetName, - MaxWireVersion: result.MaxWireVersion, + Master: result.IsMaster, + Mongos: result.Msg == "isdbgrid", + Tags: result.Tags, + SetName: result.SetName, + MaxWireVersion: result.MaxWireVersion, + MaxWriteBatchSize: result.MaxWriteBatchSize, + MaxMessageSizeBytes: result.MaxMessageSizeBytes, } hosts = make([]string, 0, 1+len(result.Hosts)+len(result.Passives)) diff --git a/cluster_test.go b/cluster_test.go index 539422be7..14d0db273 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -42,7 +42,8 @@ import ( ) func (s *S) TestNewSession(c *C) { - session, err := mgo.Dial("localhost:40001") + + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -96,7 +97,8 @@ func (s *S) TestNewSession(c *C) { } func (s *S) TestCloneSession(c *C) { - session, err := mgo.Dial("localhost:40001") + + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -161,7 +163,7 @@ func (s *S) TestCloneSession(c *C) { } func (s *S) TestModeStrong(c *C) { - session, err := mgo.Dial("localhost:40012") + session, err := mgo.Dial("localhost:40012" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -200,7 +202,7 @@ func (s *S) TestModeStrong(c *C) { func (s *S) TestModeMonotonic(c *C) { // Must necessarily connect to a slave, otherwise the // master connection will be available first. - session, err := mgo.Dial("localhost:40012") + session, err := mgo.Dial("localhost:40012" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -243,7 +245,7 @@ func (s *S) TestModeMonotonicAfterStrong(c *C) { // Test that a strong session shifting to a monotonic // one preserves the socket untouched. - session, err := mgo.Dial("localhost:40012") + session, err := mgo.Dial("localhost:40012" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -280,7 +282,7 @@ func (s *S) TestModeStrongAfterMonotonic(c *C) { // Must necessarily connect to a slave, otherwise the // master connection will be available first. - session, err := mgo.Dial("localhost:40012") + session, err := mgo.Dial("localhost:40012" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -315,7 +317,7 @@ func (s *S) TestModeStrongAfterMonotonic(c *C) { func (s *S) TestModeMonotonicWriteOnIteration(c *C) { // Must necessarily connect to a slave, otherwise the // master connection will be available first. - session, err := mgo.Dial("localhost:40012") + session, err := mgo.Dial("localhost:40012" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -360,7 +362,7 @@ func (s *S) TestModeMonotonicWriteOnIteration(c *C) { func (s *S) TestModeEventual(c *C) { // Must necessarily connect to a slave, otherwise the // master connection will be available first. - session, err := mgo.Dial("localhost:40012") + session, err := mgo.Dial("localhost:40012" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -398,7 +400,7 @@ func (s *S) TestModeEventualAfterStrong(c *C) { // Test that a strong session shifting to an eventual // one preserves the socket untouched. - session, err := mgo.Dial("localhost:40012") + session, err := mgo.Dial("localhost:40012" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -437,7 +439,7 @@ func (s *S) TestModeStrongFallover(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40021") + session, err := mgo.Dial("localhost:40021" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -478,7 +480,7 @@ func (s *S) TestModePrimaryHiccup(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40021") + session, err := mgo.Dial("localhost:40021" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -529,7 +531,7 @@ func (s *S) TestModeMonotonicFallover(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40021") + session, err := mgo.Dial("localhost:40021" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -572,7 +574,7 @@ func (s *S) TestModeMonotonicWithSlaveFallover(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40021") + session, err := mgo.Dial("localhost:40021" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -602,7 +604,7 @@ func (s *S) TestModeMonotonicWithSlaveFallover(c *C) { c.Fatal("Unknown host: ", ssresult.Host) } - session, err = mgo.Dial(addr) + session, err = mgo.Dial(addr + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -651,7 +653,7 @@ func (s *S) TestModeEventualFallover(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40021") + session, err := mgo.Dial("localhost:40021" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -688,7 +690,7 @@ func (s *S) TestModeSecondaryJustPrimary(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -703,7 +705,7 @@ func (s *S) TestModeSecondaryPreferredJustPrimary(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -719,7 +721,7 @@ func (s *S) TestModeSecondaryPreferredFallover(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40011") + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -778,7 +780,7 @@ func (s *S) TestModePrimaryPreferredFallover(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40011") + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -821,7 +823,7 @@ func (s *S) TestModePrimaryFallover(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40011") + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -848,7 +850,7 @@ func (s *S) TestModeSecondary(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40011") + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -874,7 +876,7 @@ func (s *S) TestPreserveSocketCountOnSync(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40011") + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -931,7 +933,7 @@ func (s *S) TestPreserveSocketCountOnSync(c *C) { // single connection was established. func (s *S) TestTopologySyncWithSingleMaster(c *C) { // Use hostname here rather than IP, to make things trickier. - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -955,7 +957,7 @@ func (s *S) TestTopologySyncWithSingleMaster(c *C) { func (s *S) TestTopologySyncWithSlaveSeed(c *C) { // That's supposed to be a slave. Must run discovery // and find out master to insert successfully. - session, err := mgo.Dial("localhost:40012") + session, err := mgo.Dial("localhost:40012" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -988,7 +990,7 @@ func (s *S) TestSyncTimeout(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1015,7 +1017,7 @@ func (s *S) TestDialWithTimeout(c *C) { started := time.Now() // 40009 isn't used by the test servers. - session, err := mgo.DialWithTimeout("localhost:40009", timeout) + session, err := mgo.DialWithTimeout("localhost:40009"+expFeaturesString, timeout) if session != nil { session.Close() } @@ -1030,7 +1032,7 @@ func (s *S) TestSocketTimeout(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1061,7 +1063,7 @@ func (s *S) TestSocketTimeoutOnDial(c *C) { started := time.Now() - session, err := mgo.DialWithTimeout("localhost:40001", timeout) + session, err := mgo.DialWithTimeout("localhost:40001"+expFeaturesString, timeout) c.Assert(err, ErrorMatches, "no reachable servers") c.Assert(session, IsNil) @@ -1074,7 +1076,7 @@ func (s *S) TestSocketTimeoutOnInactiveSocket(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1129,6 +1131,9 @@ func (s *S) TestDialWithReplicaSetName(c *C) { Addrs: seedList, Timeout: 5 * time.Second, ReplicaSetName: "rs1", + ExperimentalFeatures: map[string]bool{ + "opmsg": true, + }, } session, err := mgo.DialWithInfo(&info) @@ -1158,7 +1163,7 @@ func (s *S) TestDialWithReplicaSetName(c *C) { } func (s *S) TestDirect(c *C) { - session, err := mgo.Dial("localhost:40012?connect=direct") + session, err := mgo.Dial("localhost:40012?connect=direct" + "&" + string(expFeaturesString[1:])) c.Assert(err, IsNil) defer session.Close() @@ -1202,7 +1207,7 @@ func (s *S) TestDirect(c *C) { } func (s *S) TestDirectToUnknownStateMember(c *C) { - session, err := mgo.Dial("localhost:40041?connect=direct") + session, err := mgo.Dial("localhost:40041?connect=direct" + "&" + string(expFeaturesString[1:])) c.Assert(err, IsNil) defer session.Close() @@ -1229,8 +1234,11 @@ func (s *S) TestDirectToUnknownStateMember(c *C) { func (s *S) TestFailFast(c *C) { info := mgo.DialInfo{ - Addrs: []string{"localhost:99999"}, - Timeout: 5 * time.Second, + Addrs: []string{"localhost:99999"}, + Timeout: 5 * time.Second, + ExperimentalFeatures: map[string]bool{ + "opmsg": true, + }, FailFast: true, } @@ -1244,7 +1252,7 @@ func (s *S) TestFailFast(c *C) { func (s *S) countQueries(c *C, server string) (n int) { defer func() { c.Logf("Queries for %q: %d", server, n) }() - session, err := mgo.Dial(server + "?connect=direct") + session, err := mgo.Dial(server + "?connect=direct" + "&" + string(expFeaturesString[1:])) c.Assert(err, IsNil) defer session.Close() session.SetMode(mgo.Monotonic, true) @@ -1266,7 +1274,7 @@ func (s *S) countQueries(c *C, server string) (n int) { func (s *S) countCommands(c *C, server, commandName string) (n int) { defer func() { c.Logf("Queries for %q: %d", server, n) }() - session, err := mgo.Dial(server + "?connect=direct") + session, err := mgo.Dial(server + "?connect=direct" + "&" + string(expFeaturesString[1:])) c.Assert(err, IsNil) defer session.Close() session.SetMode(mgo.Monotonic, true) @@ -1284,7 +1292,7 @@ func (s *S) TestMonotonicSlaveOkFlagWithMongos(c *C) { if s.versionAtLeast(3, 4) { c.Skip("fail on 3.4+ ? ") } - session, err := mgo.Dial("localhost:40021") + session, err := mgo.Dial("localhost:40021" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1303,7 +1311,7 @@ func (s *S) TestMonotonicSlaveOkFlagWithMongos(c *C) { s.Stop(":40201") s.StartAll() - mongos, err := mgo.Dial("localhost:40202") + mongos, err := mgo.Dial("localhost:40202" + expFeaturesString) c.Assert(err, IsNil) defer mongos.Close() @@ -1378,7 +1386,7 @@ func (s *S) TestSecondaryModeWithMongos(c *C) { if s.versionAtLeast(3, 4) { c.Skip("fail on 3.4+ ?") } - session, err := mgo.Dial("localhost:40021") + session, err := mgo.Dial("localhost:40021" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1397,7 +1405,7 @@ func (s *S) TestSecondaryModeWithMongos(c *C) { s.Stop(":40201") s.StartAll() - mongos, err := mgo.Dial("localhost:40202") + mongos, err := mgo.Dial("localhost:40202" + expFeaturesString) c.Assert(err, IsNil) defer mongos.Close() @@ -1472,7 +1480,7 @@ func (s *S) TestSecondaryModeWithMongosInsert(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40202") + session, err := mgo.Dial("localhost:40202" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1493,7 +1501,7 @@ func (s *S) TestRemovalOfClusterMember(c *C) { c.Skip("-fast") } - master, err := mgo.Dial("localhost:40021") + master, err := mgo.Dial("localhost:40021" + expFeaturesString) c.Assert(err, IsNil) defer master.Close() @@ -1574,11 +1582,11 @@ func (s *S) TestPoolLimitSimple(c *C) { var session *mgo.Session var err error if test == 0 { - session, err = mgo.Dial("localhost:40001") + session, err = mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) session.SetPoolLimit(1) } else { - session, err = mgo.Dial("localhost:40001?maxPoolSize=1") + session, err = mgo.Dial("localhost:40001?maxPoolSize=1" + "&" + string(expFeaturesString[1:])) c.Assert(err, IsNil) } defer session.Close() @@ -1611,7 +1619,7 @@ func (s *S) TestPoolLimitMany(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40011") + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1650,7 +1658,7 @@ func (s *S) TestPoolLimitMany(c *C) { } func (s *S) TestSetModeEventualIterBug(c *C) { - session1, err := mgo.Dial("localhost:40011") + session1, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session1.Close() @@ -1674,7 +1682,7 @@ func (s *S) TestSetModeEventualIterBug(c *C) { } } - session2, err := mgo.Dial("localhost:40011") + session2, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session2.Close() @@ -1767,7 +1775,7 @@ func (s *S) TestPrimaryShutdownOnAuthShard(c *C) { } // Dial the shard. - session, err := mgo.Dial("localhost:40203") + session, err := mgo.Dial("localhost:40203" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1778,7 +1786,7 @@ func (s *S) TestPrimaryShutdownOnAuthShard(c *C) { c.Assert(err, IsNil) // Dial the replica set to figure the master out. - rs, err := mgo.Dial("root:rapadura@localhost:40031") + rs, err := mgo.Dial("root:rapadura@localhost:40031" + expFeaturesString) c.Assert(err, IsNil) defer rs.Close() @@ -1824,7 +1832,7 @@ func (s *S) TestNearestSecondary(c *C) { rs1c := "127.0.0.1:40013" s.Freeze(rs1b) - session, err := mgo.Dial(rs1a) + session, err := mgo.Dial(rs1a + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1889,7 +1897,7 @@ func (s *S) TestNearestServer(c *C) { rs1b := "127.0.0.1:40012" rs1c := "127.0.0.1:40013" - session, err := mgo.Dial(rs1a) + session, err := mgo.Dial(rs1a + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1953,7 +1961,7 @@ func (s *S) TestConnectCloseConcurrency(c *C) { for i := 0; i < n; i++ { go func() { defer wg.Done() - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) if err != nil { c.Fatal(err) } @@ -1969,7 +1977,7 @@ func (s *S) TestSelectServers(c *C) { c.Skip("read preferences introduced in 2.2") } - session, err := mgo.Dial("localhost:40011") + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1998,7 +2006,7 @@ func (s *S) TestSelectServersWithMongos(c *C) { c.Skip("fail on 3.4+") } - session, err := mgo.Dial("localhost:40021") + session, err := mgo.Dial("localhost:40021" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -2029,7 +2037,7 @@ func (s *S) TestSelectServersWithMongos(c *C) { q23a := s.countQueries(c, "localhost:40023") // Do a SlaveOk query through MongoS - mongos, err := mgo.Dial("localhost:40202") + mongos, err := mgo.Dial("localhost:40202" + expFeaturesString) c.Assert(err, IsNil) defer mongos.Close() @@ -2083,11 +2091,11 @@ func (s *S) TestDoNotFallbackToMonotonic(c *C) { if !s.versionAtLeast(3, 0) { c.Skip("command-counting logic depends on 3.0+") } - if s.versionAtLeast(3, 4) { - c.Skip("failing on 3.4+") + if s.versionAtLeast(3, 2, 17) { + c.Skip("failing on 3.2.17+") } - session, err := mgo.Dial("localhost:40012") + session, err := mgo.Dial("localhost:40012" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() diff --git a/gridfs_test.go b/gridfs_test.go index 9fdd0a26f..984ded17c 100644 --- a/gridfs_test.go +++ b/gridfs_test.go @@ -184,7 +184,7 @@ func (s *S) TestGridFSFileDetails(c *C) { } func (s *S) TestGridFSSetUploadDate(c *C) { - session, err := mgo.Dial("localhost:40011") + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -212,7 +212,7 @@ func (s *S) TestGridFSSetUploadDate(c *C) { } func (s *S) TestGridFSCreateWithChunking(c *C) { - session, err := mgo.Dial("localhost:40011") + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -293,7 +293,7 @@ func (s *S) TestGridFSCreateWithChunking(c *C) { } func (s *S) TestGridFSAbort(c *C) { - session, err := mgo.Dial("localhost:40011") + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -330,7 +330,7 @@ func (s *S) TestGridFSAbort(c *C) { } func (s *S) TestGridFSCloseConflict(c *C) { - session, err := mgo.Dial("localhost:40011") + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -358,7 +358,7 @@ func (s *S) TestGridFSCloseConflict(c *C) { } func (s *S) TestGridFSOpenNotFound(c *C) { - session, err := mgo.Dial("localhost:40011") + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -375,7 +375,7 @@ func (s *S) TestGridFSOpenNotFound(c *C) { } func (s *S) TestGridFSReadAll(c *C) { - session, err := mgo.Dial("localhost:40011") + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -412,7 +412,7 @@ func (s *S) TestGridFSReadAll(c *C) { } func (s *S) TestGridFSReadChunking(c *C) { - session, err := mgo.Dial("localhost:40011") + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -472,7 +472,7 @@ func (s *S) TestGridFSReadChunking(c *C) { } func (s *S) TestGridFSOpen(c *C) { - session, err := mgo.Dial("localhost:40011") + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -502,7 +502,7 @@ func (s *S) TestGridFSOpen(c *C) { } func (s *S) TestGridFSSeek(c *C) { - session, err := mgo.Dial("localhost:40011") + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -581,7 +581,7 @@ func (s *S) TestGridFSSeek(c *C) { } func (s *S) TestGridFSRemoveId(c *C) { - session, err := mgo.Dial("localhost:40011") + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -619,7 +619,7 @@ func (s *S) TestGridFSRemoveId(c *C) { } func (s *S) TestGridFSRemove(c *C) { - session, err := mgo.Dial("localhost:40011") + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -649,7 +649,7 @@ func (s *S) TestGridFSRemove(c *C) { } func (s *S) TestGridFSOpenNext(c *C) { - session, err := mgo.Dial("localhost:40011") + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() diff --git a/harness/daemons/.env b/harness/daemons/.env index b9a900647..7ba8cf599 100644 --- a/harness/daemons/.env +++ b/harness/daemons/.env @@ -22,10 +22,8 @@ versionAtLeast() { COMMONDOPTSNOIP=" --nohttpinterface - --noprealloc --nojournal --smallfiles - --nssize=1 --oplogSize=1 --dbpath ./db " @@ -55,14 +53,12 @@ if versionAtLeast 3 2; then # 3.2 doesn't like --nojournal on config servers. COMMONCOPTS="$(echo "$COMMONCOPTS" | sed '/--nojournal/d')" - if versionAtLeast 3 4; then # http interface is disabled by default, this option does not exist anymore COMMONDOPTSNOIP="$(echo "$COMMONDOPTSNOIP" | sed '/--nohttpinterface/d')" COMMONDOPTS="$(echo "$COMMONDOPTS" | sed '/--nohttpinterface/d')" COMMONCOPTS="$(echo "$COMMONCOPTS" | sed '/--nohttpinterface/d')" - # config server need to be started as replica set CFG1OPTS="--replSet conf1" CFG2OPTS="--replSet conf2" @@ -71,12 +67,6 @@ if versionAtLeast 3 2; then MONGOS1OPTS="--configdb conf1/127.0.0.1:40101" MONGOS2OPTS="--configdb conf2/127.0.0.1:40102" MONGOS3OPTS="--configdb conf3/127.0.0.1:40103" - else - - # Go back to MMAPv1 so it's not super sluggish. :-( - COMMONDOPTSNOIP="--storageEngine=mmapv1 $COMMONDOPTSNOIP" - COMMONDOPTS="--storageEngine=mmapv1 $COMMONDOPTS" - COMMONCOPTS="--storageEngine=mmapv1 $COMMONCOPTS" fi fi diff --git a/harness/daemons/db2/run b/harness/daemons/db2/run index 5c7b1aa50..e0ec381b3 100755 --- a/harness/daemons/db2/run +++ b/harness/daemons/db2/run @@ -3,6 +3,5 @@ . ../.env exec mongod $COMMONDOPTS \ - --shardsvr \ --port 40002 \ --auth diff --git a/harness/daemons/db3/run b/harness/daemons/db3/run index 539da5fb2..a3788734e 100755 --- a/harness/daemons/db3/run +++ b/harness/daemons/db3/run @@ -3,7 +3,6 @@ . ../.env exec mongod $COMMONDOPTS \ - --shardsvr \ --port 40003 \ --auth \ --sslMode preferSSL \ diff --git a/harness/mongojs/dropall.js b/harness/mongojs/dropall.js index 7fa39d112..ebdf820ef 100644 --- a/harness/mongojs/dropall.js +++ b/harness/mongojs/dropall.js @@ -16,26 +16,33 @@ for (var i in ports) { for (var j in auth) { if (auth[j] == port) { - admin.auth("root", "rapadura") - admin.system.users.find().forEach(function(u) { - if (u.user == "root" || u.user == "reader") { - return; + print("removing user for port " + auth[j]) + for (var k = 0; k < 10; k++) { + var ok = admin.auth("root", "rapadura") + if (ok) { + admin.system.users.find().forEach(function (u) { + if (u.user == "root" || u.user == "reader") { + return; + } + if (typeof admin.dropUser == "function") { + mongo.getDB(u.db).dropUser(u.user); + } else { + admin.removeUser(u.user); + } + }) + break } - if (typeof admin.dropUser == "function") { - mongo.getDB(u.db).dropUser(u.user); - } else { - admin.removeUser(u.user); - } - }) - break + print("failed to auth for port " + port + " retrying in 1s ") + sleep(1000) + } } } - var result = admin.runCommand({"listDatabases": 1}) + var result = admin.runCommand({ "listDatabases": 1 }) for (var j = 0; j != 100; j++) { if (typeof result.databases != "undefined" || notMaster(result)) { break } - result = admin.runCommand({"listDatabases": 1}) + result = admin.runCommand({ "listDatabases": 1 }) } if (notMaster(result)) { continue @@ -49,18 +56,18 @@ for (var i in ports) { for (var j = 0; j != dbs.length; j++) { var db = dbs[j] switch (db.name) { - case "admin": - case "local": - case "config": - break - default: - mongo.getDB(db.name).dropDatabase() + case "admin": + case "local": + case "config": + break + default: + mongo.getDB(db.name).dropDatabase() } } } function notMaster(result) { - return typeof result.errmsg != "undefined" && (result.errmsg.indexOf("not master") >= 0 || result.errmsg.indexOf("no master found")) + return typeof result.errmsg != "undefined" && (result.errmsg.indexOf("not master") >= 0 || result.errmsg.indexOf("no master found")) } // vim:ts=4:sw=4:et diff --git a/harness/mongojs/init.js b/harness/mongojs/init.js index 909cf5162..a5ee1c0e8 100644 --- a/harness/mongojs/init.js +++ b/harness/mongojs/init.js @@ -2,37 +2,42 @@ var settings = {}; // We know the master of the first set (pri=1), but not of the second. -var rs1cfg = {_id: "rs1", - members: [{_id: 1, host: "127.0.0.1:40011", priority: 1, tags: {rs1: "a"}}, - {_id: 2, host: "127.0.0.1:40012", priority: 0, tags: {rs1: "b"}}, - {_id: 3, host: "127.0.0.1:40013", priority: 0, tags: {rs1: "c"}}], - settings: settings} -var rs2cfg = {_id: "rs2", - members: [{_id: 1, host: "127.0.0.1:40021", priority: 1, tags: {rs2: "a"}}, - {_id: 2, host: "127.0.0.1:40022", priority: 1, tags: {rs2: "b"}}, - {_id: 3, host: "127.0.0.1:40023", priority: 1, tags: {rs2: "c"}}], - settings: settings} -var rs3cfg = {_id: "rs3", - members: [{_id: 1, host: "127.0.0.1:40031", priority: 1, tags: {rs3: "a"}}, - {_id: 2, host: "127.0.0.1:40032", priority: 1, tags: {rs3: "b"}}, - {_id: 3, host: "127.0.0.1:40033", priority: 1, tags: {rs3: "c"}}], - settings: settings} +var rs1cfg = { + _id: "rs1", + members: [{ _id: 1, host: "127.0.0.1:40011", priority: 1, tags: { rs1: "a" } }, + { _id: 2, host: "127.0.0.1:40012", priority: 0, tags: { rs1: "b" } }, + { _id: 3, host: "127.0.0.1:40013", priority: 0, tags: { rs1: "c" } }], + settings: settings +} +var rs2cfg = { + _id: "rs2", + members: [{ _id: 1, host: "127.0.0.1:40021", priority: 1, tags: { rs2: "a" } }, + { _id: 2, host: "127.0.0.1:40022", priority: 1, tags: { rs2: "b" } }, + { _id: 3, host: "127.0.0.1:40023", priority: 1, tags: { rs2: "c" } }], + settings: settings +} +var rs3cfg = { + _id: "rs3", + members: [{ _id: 1, host: "127.0.0.1:40031", priority: 1, tags: { rs3: "a" } }, + { _id: 2, host: "127.0.0.1:40032", priority: 0, tags: { rs3: "b" } }, + { _id: 3, host: "127.0.0.1:40033", priority: 0, tags: { rs3: "c" } }], + settings: settings +} for (var i = 0; i != 60; i++) { - try { - db1 = new Mongo("127.0.0.1:40001").getDB("admin") - db2 = new Mongo("127.0.0.1:40002").getDB("admin") - rs1a = new Mongo("127.0.0.1:40011").getDB("admin") - rs2a = new Mongo("127.0.0.1:40021").getDB("admin") - rs3a = new Mongo("127.0.0.1:40031").getDB("admin") + try { + db1 = new Mongo("127.0.0.1:40001").getDB("admin") + rs1a = new Mongo("127.0.0.1:40011").getDB("admin") + rs2a = new Mongo("127.0.0.1:40021").getDB("admin") + rs3a = new Mongo("127.0.0.1:40031").getDB("admin") cfg1 = new Mongo("127.0.0.1:40101").getDB("admin") cfg2 = new Mongo("127.0.0.1:40102").getDB("admin") cfg3 = new Mongo("127.0.0.1:40103").getDB("admin") - break - } catch(err) { - print("Can't connect yet...") - } - sleep(1000) + break + } catch (err) { + print("Can't connect yet...") + } + sleep(1000) } function hasSSL() { @@ -52,28 +57,19 @@ function versionAtLeast() { return true } -rs1a.runCommand({replSetInitiate: rs1cfg}) -rs2a.runCommand({replSetInitiate: rs2cfg}) -rs3a.runCommand({replSetInitiate: rs3cfg}) -if (versionAtLeast(3,4)) { - print("configuring config server for mongodb 3.4") - cfg1.runCommand({replSetInitiate: {_id:"conf1", members: [{"_id":1, "host":"localhost:40101"}]}}) - cfg2.runCommand({replSetInitiate: {_id:"conf2", members: [{"_id":1, "host":"localhost:40102"}]}}) - cfg3.runCommand({replSetInitiate: {_id:"conf3", members: [{"_id":1, "host":"localhost:40103"}]}}) +if (versionAtLeast(3, 4)) { + print("configuring config server for mongodb 3.4+") + cfg1.runCommand({ replSetInitiate: { _id: "conf1", configsvr: true, members: [{ "_id": 1, "host": "localhost:40101" }] } }) + cfg2.runCommand({ replSetInitiate: { _id: "conf2", configsvr: true, members: [{ "_id": 1, "host": "localhost:40102" }] } }) + cfg3.runCommand({ replSetInitiate: { _id: "conf3", configsvr: true, members: [{ "_id": 1, "host": "localhost:40103" }] } }) } -function configShards() { - s1 = new Mongo("127.0.0.1:40201").getDB("admin") - s1.runCommand({addshard: "127.0.0.1:40001"}) - s1.runCommand({addshard: "rs1/127.0.0.1:40011"}) - - s2 = new Mongo("127.0.0.1:40202").getDB("admin") - s2.runCommand({addshard: "rs2/127.0.0.1:40021"}) +sleep(3000) - s3 = new Mongo("127.0.0.1:40203").getDB("admin") - s3.runCommand({addshard: "rs3/127.0.0.1:40031"}) -} +rs1a.runCommand({ replSetInitiate: rs1cfg }) +rs2a.runCommand({ replSetInitiate: rs2cfg }) +rs3a.runCommand({ replSetInitiate: rs3cfg }) function configAuth() { var addrs = ["127.0.0.1:40002", "127.0.0.1:40203", "127.0.0.1:40031"] @@ -83,21 +79,26 @@ function configAuth() { for (var i in addrs) { print("Configuring auth for", addrs[i]) var db = new Mongo(addrs[i]).getDB("admin") - var v = db.serverBuildInfo().versionArray var timedOut = false - if (v < [2, 5]) { - db.addUser("root", "rapadura") - } else { + createUser: + for (var i = 0; i < 60; i++) { try { - db.createUser({user: "root", pwd: "rapadura", roles: ["root"]}) + db.createUser({ user: "root", pwd: "rapadura", roles: ["root"] }) } catch (err) { // 3.2 consistently fails replication of creds on 40031 (config server) print("createUser command returned an error: " + err) if (String(err).indexOf("timed out") >= 0) { timedOut = true; } + // on 3.6 cluster with keyFile, we sometimes get this error + if (String(err).indexOf("Cache Reader No keys found for HMAC that is valid for time")) { + sleep(500) + continue createUser; + } } + break; } + for (var i = 0; i < 60; i++) { var ok = db.auth("root", "rapadura") if (ok || !timedOut) { @@ -105,18 +106,47 @@ function configAuth() { } sleep(1000); } - if (v >= [2, 6]) { - db.createUser({user: "reader", pwd: "rapadura", roles: ["readAnyDatabase"]}) - } else if (v >= [2, 4]) { - db.addUser({user: "reader", pwd: "rapadura", roles: ["readAnyDatabase"]}) - } else { - db.addUser("reader", "rapadura", true) + sleep(500) + db.createUser({ user: "reader", pwd: "rapadura", roles: ["readAnyDatabase"] }) + sleep(3000) + } +} + +function addShard(adminDb, shardList) { + for (var index = 0; index < shardList.length; index++) { + for (var i = 0; i < 10; i++) { + var result = adminDb.runCommand({ addshard: shardList[index] }) + if (result.ok == 1) { + print("shard " + shardList[index] + " sucessfully added") + break + } else { + print("fail to add shard: " + shardList[index] + " error: " + JSON.stringify(result) + ", retrying in 1s") + sleep(1000) + } } } } +function configShards() { + s1 = new Mongo("127.0.0.1:40201").getDB("admin") + addShard(s1, ["127.0.0.1:40001", "rs1/127.0.0.1:40011"]) + + s2 = new Mongo("127.0.0.1:40202").getDB("admin") + addShard(s2, ["rs2/127.0.0.1:40021"]) + + s3 = new Mongo("127.0.0.1:40203").getDB("admin") + for (var i = 0; i < 10; i++) { + var ok = s3.auth("root", "rapadura") + if (ok) { + break + } + sleep(1000) + } + addShard(s3, ["rs3/127.0.0.1:40031"]) +} + function countHealthy(rs) { - var status = rs.runCommand({replSetGetStatus: 1}) + var status = rs.runCommand({ replSetGetStatus: 1 }) var count = 0 var primary = 0 if (typeof status.members != "undefined") { @@ -131,7 +161,7 @@ function countHealthy(rs) { } } if (primary == 0) { - count = 0 + count = 0 } return count } @@ -142,8 +172,9 @@ for (var i = 0; i != 60; i++) { var count = countHealthy(rs1a) + countHealthy(rs2a) + countHealthy(rs3a) print("Replica sets have", count, "healthy nodes.") if (count == totalRSMembers) { - configShards() configAuth() + sleep(2000) + configShards() quit(0) } sleep(1000) diff --git a/server.go b/server.go index 7ad955255..ce6140941 100644 --- a/server.go +++ b/server.go @@ -36,6 +36,12 @@ import ( "github.com/globalsign/mgo/bson" ) +const ( + // default value for MongoDB 3.6 + defaultWriteBatchSize = 100000 + defaultMaxMessageSizeBytes = 48000000 +) + // --------------------------------------------------------------------------- // Mongo server encapsulation. @@ -67,15 +73,15 @@ func (dial dialer) isSet() bool { } type mongoServerInfo struct { - Master bool - Mongos bool - Tags bson.D - MaxWireVersion int - SetName string + Master bool + Mongos bool + Tags bson.D + MaxWireVersion int + SetName string + MaxWriteBatchSize int + MaxMessageSizeBytes int } -var defaultServerInfo mongoServerInfo - func newServer(addr string, tcpaddr *net.TCPAddr, sync chan bool, dial dialer) *mongoServer { server := &mongoServer{ Addr: addr, @@ -83,8 +89,11 @@ func newServer(addr string, tcpaddr *net.TCPAddr, sync chan bool, dial dialer) * tcpaddr: tcpaddr, sync: sync, dial: dial, - info: &defaultServerInfo, - pingValue: time.Hour, // Push it back before an actual ping. + info: &mongoServerInfo{ + MaxWriteBatchSize: defaultWriteBatchSize, + MaxMessageSizeBytes: defaultMaxMessageSizeBytes, + }, + pingValue: time.Hour, // Push it back before an actual ping. } go server.pinger(true) return server diff --git a/session.go b/session.go index 91adfab67..cdce36919 100644 --- a/session.go +++ b/session.go @@ -103,6 +103,7 @@ type Session struct { queryConfig query bypassValidation bool slaveOk bool + experimental map[string]bool } // Database holds collections of documents @@ -319,6 +320,7 @@ func ParseURL(url string) (*DialInfo, error) { poolLimit := 0 appName := "" readPreferenceMode := Primary + experimental := map[string]bool{} var readPreferenceTagSets []bson.D for _, opt := range uinfo.options { switch opt.key { @@ -340,6 +342,13 @@ func ParseURL(url string) (*DialInfo, error) { return nil, errors.New("appName too long, must be < 128 bytes: " + opt.value) } appName = opt.value + case "experimental": + switch opt.value { + case "opmsg": + experimental[opt.value] = true + default: + return nil, errors.New("unknow experimental feature: " + opt.value) + } case "readPreference": switch opt.value { case "nearest": @@ -399,7 +408,8 @@ func ParseURL(url string) (*DialInfo, error) { Mode: readPreferenceMode, TagSets: readPreferenceTagSets, }, - ReplicaSetName: setName, + ReplicaSetName: setName, + ExperimentalFeatures: experimental, } return &info, nil } @@ -479,6 +489,12 @@ type DialInfo struct { // WARNING: This field is obsolete. See DialServer above. Dial func(addr net.Addr) (net.Conn, error) + + // List of experimental feature to enable. Set the value to 'true' + // to enable a feature. + // Currently, experimental features are: + // - opmsg + ExperimentalFeatures map[string]bool } // ReadPreference defines the manner in which servers are chosen. @@ -569,7 +585,12 @@ func DialWithInfo(info *DialInfo) (*Session, error) { } else { session.SetMode(Strong, true) } - + if len(info.ExperimentalFeatures) > 0 { + session.experimental = make(map[string]bool, 0) + for k, v := range info.ExperimentalFeatures { + session.experimental[k] = v + } + } return session, nil } @@ -661,6 +682,14 @@ func copySession(session *Session, keepCreds bool) (s *Session) { } else if session.dialCred != nil { creds = []Credential{*session.dialCred} } + + var experimental map[string]bool + if len(session.experimental) > 0 { + experimental = make(map[string]bool, len(session.experimental)) + for k, v := range session.experimental { + experimental[k] = v + } + } scopy := Session{ defaultdb: session.defaultdb, sourcedb: session.sourcedb, @@ -678,6 +707,7 @@ func copySession(session *Session, keepCreds bool) (s *Session) { queryConfig: session.queryConfig, bypassValidation: session.bypassValidation, slaveOk: session.slaveOk, + experimental: experimental, } s = &scopy debugf("New session %p on cluster %p (copy from %p)", s, cluster, session) @@ -4926,13 +4956,15 @@ func (iter *Iter) replyFunc() replyFunc { } type writeCmdResult struct { - Ok bool - N int - NModified int `bson:"nModified"` + Ok bool `bson:"ok"` + N int `bson:"n"` + NModified int `bson:"nModified"` Upserted []struct { Index int Id interface{} `bson:"_id"` } + Code int `bson:"code"` + Errmsg string `bson:"errmsg"` ConcernError writeConcernError `bson:"writeConcernError"` Errors []writeCmdError `bson:"writeErrors"` } @@ -4956,6 +4988,140 @@ func (r *writeCmdResult) BulkErrorCases() []BulkErrorCase { return ecases } +func (c *Collection) writeOpWithOpMsg(socket *mongoSocket, serverInfo *mongoServerInfo, op interface{}, ordered, bypassValidation bool, safeOp *queryOp) (*LastError, error) { + var cmd bson.D + var documents []interface{} + var docSeqID string + canUseOpMsg := true + switch msgOp := op.(type) { + case *insertOp: + cmd = bson.D{ + {Name: "insert", Value: c.Name}, + } + docSeqID = "documents" + documents = msgOp.documents + case bulkUpdateOp: + cmd = bson.D{ + {Name: "update", Value: c.Name}, + } + docSeqID = "updates" + documents = msgOp + case bulkDeleteOp: + cmd = bson.D{ + {Name: "delete", Value: c.Name}, + } + docSeqID = "deletes" + documents = msgOp + default: + canUseOpMsg = false + } + + if canUseOpMsg { + //msg flags, see https://docs.mongodb.com/master/reference/mongodb-wire-protocol/#flag-bits + flags := uint32(0) + + var writeConcern interface{} + if safeOp == nil { + // unacknowledged writes + flags |= opMsgFlagMoreToCome + writeConcern = bson.D{{Name: "w", Value: 0}} + } else { + writeConcern = safeOp.query.(*getLastError) + } + + cmd = append(cmd, bson.DocElem{ + Name: "$db", Value: c.Database.Name, + }, bson.DocElem{ + Name: "ordered", Value: ordered, + }, bson.DocElem{ + Name: "writeConcern", Value: writeConcern, + }, bson.DocElem{ + Name: "bypassDocumentValidation", Value: bypassValidation, + }) + + body := msgSection{ + payloadType: msgPayload0, + data: cmd, + } + + n := 0 + modified := 0 + var errs []BulkErrorCase + var lerr LastError + + l := len(documents) + batchNb := (l / serverInfo.MaxWriteBatchSize) + 1 + if l != 0 && (l%serverInfo.MaxWriteBatchSize) == 0 { + batchNb-- + } + count := 0 + + for count < batchNb { + start := count * serverInfo.MaxWriteBatchSize + length := l - start + if length > serverInfo.MaxWriteBatchSize { + length = serverInfo.MaxWriteBatchSize + } + + docs := msgSection{ + payloadType: msgPayload1, + data: payloadType1{ + identifier: docSeqID, + docs: documents[start : start+length], + }, + } + count++ + + // CRC-32 checksum is not implemented in Mongodb 3.6 but + // will be in future release. It's optional, so no need + // to set it for the moment + newOp := &msgOp{ + flags: flags, + sections: []msgSection{body, docs}, + checksum: 0, + } + result, err := socket.sendMessage(newOp) + if err != nil { + return &lerr, err + } + // for some reason, command result format has changed and + // code|errmsg are sometimes top level fields in writeCommandResult + // TODO need to investigate further + if result.Code != 0 { + return &lerr, errors.New(result.Errmsg) + } + if result.ConcernError.Code != 0 { + return &lerr, errors.New(result.ConcernError.ErrMsg) + } + + n += result.N + modified += result.NModified + + if len(result.Errors) > 0 { + for _, e := range result.Errors { + errs = append(errs, BulkErrorCase{ + e.Index, + &QueryError{ + Code: e.Code, + Message: e.ErrMsg, + }, + }) + } + } + } + lerr = LastError{ + N: n, + modified: modified, + ecases: errs, + } + if len(lerr.ecases) > 0 { + return &lerr, lerr.ecases[0].Err + } + return &lerr, nil + } + return nil, nil +} + // writeOp runs the given modifying operation, potentially followed up // by a getLastError command in case the session is in safe mode. The // LastError result is made available in lerr, and if lerr.Err is set it @@ -4971,9 +5137,20 @@ func (c *Collection) writeOp(op interface{}, ordered bool) (lerr *LastError, err s.m.RLock() safeOp := s.safeOp bypassValidation := s.bypassValidation + enableOpMsg := s.experimental["opmsg"] s.m.RUnlock() - if socket.ServerInfo().MaxWireVersion >= 2 { + serverInfo := socket.ServerInfo() + + if enableOpMsg && serverInfo.MaxWireVersion >= 6 { + // we can use OP_MSG introduced in Mongodb 3.6 + oPlerr, oPerr := c.writeOpWithOpMsg(socket, serverInfo, op, ordered, bypassValidation, safeOp) + if oPlerr != nil || oPerr != nil { + return oPlerr, oPerr + } + } + + if serverInfo.MaxWireVersion >= 2 { // Servers with a more recent write protocol benefit from write commands. if op, ok := op.(*insertOp); ok && len(op.documents) > 1000 { var lerr LastError @@ -5253,7 +5430,7 @@ func getRFC2253NameString(RDNElements *pkix.RDNSequence) string { var replacer = strings.NewReplacer(",", "\\,", "=", "\\=", "+", "\\+", "<", "\\<", ">", "\\>", ";", "\\;") //The elements in the sequence needs to be reversed when converting them for i := len(*RDNElements) - 1; i >= 0; i-- { - var nameAndValueList = make([]string,len((*RDNElements)[i])) + var nameAndValueList = make([]string, len((*RDNElements)[i])) for j, attribute := range (*RDNElements)[i] { var shortAttributeName = rdnOIDToShortName(attribute.Type) if len(shortAttributeName) <= 0 { diff --git a/session_test.go b/session_test.go index 227052719..84aa1f8a7 100644 --- a/session_test.go +++ b/session_test.go @@ -44,7 +44,7 @@ import ( ) func (s *S) TestRunString(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -55,7 +55,7 @@ func (s *S) TestRunString(c *C) { } func (s *S) TestRunValue(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -66,7 +66,7 @@ func (s *S) TestRunValue(c *C) { } func (s *S) TestPing(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -86,19 +86,19 @@ func (s *S) TestPing(c *C) { } func (s *S) TestDialIPAddress(c *C) { - session, err := mgo.Dial("127.0.0.1:40001") + session, err := mgo.Dial("127.0.0.1:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() if os.Getenv("NOIPV6") != "1" { - session, err = mgo.Dial("[::1%]:40001") + session, err = mgo.Dial("[::1%]:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() } } func (s *S) TestURLSingle(c *C) { - session, err := mgo.Dial("mongodb://localhost:40001/") + session, err := mgo.Dial("mongodb://localhost:40001/" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -109,7 +109,7 @@ func (s *S) TestURLSingle(c *C) { } func (s *S) TestURLMany(c *C) { - session, err := mgo.Dial("mongodb://localhost:40011,localhost:40012/") + session, err := mgo.Dial("mongodb://localhost:40011,localhost:40012/" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -125,7 +125,7 @@ func (s *S) TestURLParsing(c *C) { "localhost:40001?foo=1;bar=2", } for _, url := range urls { - session, err := mgo.Dial(url) + session, err := mgo.Dial(url + "&" + string(expFeaturesString[1:])) if session != nil { session.Close() } @@ -205,7 +205,7 @@ func (s *S) TestURLWithAppName(c *C) { c.Skip("appName depends on MongoDB 3.4+") } appName := "myAppName" - session, err := mgo.Dial("localhost:40001?appName=" + appName) + session, err := mgo.Dial("localhost:40001?appName=" + appName + "&" + string(expFeaturesString[1:])) c.Assert(err, IsNil) defer session.Close() @@ -240,12 +240,12 @@ func (s *S) TestURLWithAppNameTooLong(c *C) { } appName := "myAppNameWayTooLongmyAppNameWayTooLongmyAppNameWayTooLongmyAppNameWayTooLong" appName += appName - _, err := mgo.Dial("localhost:40001?appName=" + appName) + _, err := mgo.Dial("localhost:40001?appName=" + appName + "&" + string(expFeaturesString[1:])) c.Assert(err, ErrorMatches, "appName too long, must be < 128 bytes: "+appName) } func (s *S) TestInsertFindOne(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -269,7 +269,7 @@ func (s *S) TestInsertFindOne(c *C) { } func (s *S) TestInsertFindOneNil(c *C) { - session, err := mgo.Dial("localhost:40002") + session, err := mgo.Dial("localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -279,7 +279,7 @@ func (s *S) TestInsertFindOneNil(c *C) { } func (s *S) TestInsertFindOneMap(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -294,7 +294,7 @@ func (s *S) TestInsertFindOneMap(c *C) { } func (s *S) TestInsertFindAll(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -340,7 +340,7 @@ func (s *S) TestInsertFindAll(c *C) { } func (s *S) TestFindRef(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -386,7 +386,7 @@ func (s *S) TestFindRef(c *C) { } func (s *S) TestDatabaseAndCollectionNames(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -413,23 +413,15 @@ func (s *S) TestDatabaseAndCollectionNames(c *C) { names, err = db1.CollectionNames() c.Assert(err, IsNil) - if s.versionAtLeast(3, 4) { - c.Assert(names, DeepEquals, []string{"col1", "col2"}) - } else { - c.Assert(names, DeepEquals, []string{"col1", "col2", "system.indexes"}) - } + c.Assert(filterDBs(names), DeepEquals, []string{"col1", "col2"}) names, err = db2.CollectionNames() c.Assert(err, IsNil) - if s.versionAtLeast(3, 4) { - c.Assert(names, DeepEquals, []string{"col3"}) - } else { - c.Assert(names, DeepEquals, []string{"col3", "system.indexes"}) - } + c.Assert(filterDBs(names), DeepEquals, []string{"col3"}) } func (s *S) TestSelect(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -445,7 +437,7 @@ func (s *S) TestSelect(c *C) { } func (s *S) TestInlineMap(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -476,7 +468,7 @@ func (s *S) TestInlineMap(c *C) { } func (s *S) TestUpdate(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -508,7 +500,7 @@ func (s *S) TestUpdate(c *C) { } func (s *S) TestUpdateId(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -536,7 +528,7 @@ func (s *S) TestUpdateId(c *C) { } func (s *S) TestUpdateNil(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -566,7 +558,7 @@ func (s *S) TestUpdateNil(c *C) { } func (s *S) TestUpsert(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -629,7 +621,7 @@ func (s *S) TestUpsert(c *C) { } func (s *S) TestUpsertId(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -666,7 +658,7 @@ func (s *S) TestUpsertId(c *C) { } func (s *S) TestUpdateAll(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -715,7 +707,7 @@ func (s *S) TestUpdateAll(c *C) { } func (s *S) TestRemove(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -744,7 +736,7 @@ func (s *S) TestRemove(c *C) { } func (s *S) TestRemoveId(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -762,7 +754,7 @@ func (s *S) TestRemoveId(c *C) { } func (s *S) TestRemoveUnsafe(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -782,7 +774,7 @@ func (s *S) TestRemoveUnsafe(c *C) { } func (s *S) TestRemoveAll(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -825,7 +817,7 @@ func (s *S) TestRemoveAll(c *C) { } func (s *S) TestDropDatabase(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -854,7 +846,7 @@ func filterDBs(dbs []string) []string { var i int for _, name := range dbs { switch name { - case "admin", "local": + case "admin", "local", "config", "system.indexes": default: dbs[i] = name i++ @@ -867,7 +859,7 @@ func filterDBs(dbs []string) []string { } func (s *S) TestDropCollection(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -880,26 +872,18 @@ func (s *S) TestDropCollection(c *C) { names, err := db.CollectionNames() c.Assert(err, IsNil) - if s.versionAtLeast(3, 4) { - c.Assert(names, DeepEquals, []string{"col2"}) - } else { - c.Assert(names, DeepEquals, []string{"col2", "system.indexes"}) - } + c.Assert(filterDBs(names), DeepEquals, []string{"col2"}) err = db.C("col2").DropCollection() c.Assert(err, IsNil) names, err = db.CollectionNames() c.Assert(err, IsNil) - if s.versionAtLeast(3, 4) { - c.Assert(len(names), Equals, 0) - } else { - c.Assert(names, DeepEquals, []string{"system.indexes"}) - } + c.Assert(len(filterDBs(names)), Equals, 0) } func (s *S) TestCreateCollectionCapped(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -925,7 +909,7 @@ func (s *S) TestCreateCollectionCapped(c *C) { } func (s *S) TestCreateCollectionNoIndex(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -945,7 +929,7 @@ func (s *S) TestCreateCollectionNoIndex(c *C) { } func (s *S) TestCreateCollectionForceIndex(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -971,7 +955,7 @@ func (s *S) TestCreateCollectionValidator(c *C) { if !s.versionAtLeast(3, 2) { c.Skip("validation depends on MongoDB 3.2+") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1023,7 +1007,7 @@ func (s *S) TestCreateCollectionStorageEngine(c *C) { if !s.versionAtLeast(3, 0) { c.Skip("storageEngine option depends on MongoDB 3.0+") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1041,7 +1025,7 @@ func (s *S) TestCreateCollectionWithCollation(c *C) { if !s.versionAtLeast(3, 4) { c.Skip("depends on mongodb 3.4+") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1085,7 +1069,7 @@ func (s *S) TestIsDupValues(c *C) { } func (s *S) TestIsDupPrimary(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1099,7 +1083,7 @@ func (s *S) TestIsDupPrimary(c *C) { } func (s *S) TestIsDupUnique(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1121,7 +1105,7 @@ func (s *S) TestIsDupUnique(c *C) { } func (s *S) TestIsDupCapped(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1145,7 +1129,7 @@ func (s *S) TestIsDupCapped(c *C) { } func (s *S) TestIsDupFindAndModify(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1164,7 +1148,7 @@ func (s *S) TestIsDupFindAndModify(c *C) { } func (s *S) TestIsDupRetryUpsert(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1184,7 +1168,7 @@ func (s *S) TestIsDupRetryUpsert(c *C) { } func (s *S) TestFindAndModify(c *C) { - session, err := mgo.Dial("localhost:40011") + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1252,7 +1236,7 @@ func (s *S) TestFindAndModify(c *C) { } func (s *S) TestFindAndModifyBug997828(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1263,24 +1247,18 @@ func (s *S) TestFindAndModifyBug997828(c *C) { result := make(M) _, err = coll.Find(M{"n": "not-a-number"}).Apply(mgo.Change{Update: M{"$inc": M{"n": 1}}}, result) c.Assert(err, ErrorMatches, `(exception: )?Cannot apply \$inc .*`) - if s.versionAtLeast(2, 1) { - qerr, _ := err.(*mgo.QueryError) - c.Assert(qerr, NotNil, Commentf("err: %#v", err)) - if s.versionAtLeast(2, 6) { - // Oh, the dance of error codes. :-( - c.Assert(qerr.Code, Equals, 16837) - } else { - c.Assert(qerr.Code, Equals, 10140) - } + qerr, _ := err.(*mgo.QueryError) + c.Assert(qerr, NotNil, Commentf("err: %#v", err)) + if s.versionAtLeast(3, 6) { + // Oh, the dance of error codes. :-( + c.Assert(qerr.Code, Equals, 14) } else { - lerr, _ := err.(*mgo.LastError) - c.Assert(lerr, NotNil, Commentf("err: %#v", err)) - c.Assert(lerr.Code, Equals, 10140) + c.Assert(qerr.Code, Equals, 16837) } } func (s *S) TestFindAndModifyErrmsgDoc(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1294,7 +1272,7 @@ func (s *S) TestFindAndModifyErrmsgDoc(c *C) { } func (s *S) TestCountCollection(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1316,7 +1294,7 @@ func (s *S) TestView(c *C) { c.Skip("depends on mongodb 3.4+") } // CreateView has to be run against mongos - session, err := mgo.Dial("localhost:40201") + session, err := mgo.Dial("localhost:40201" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1404,16 +1382,12 @@ func (s *S) TestView(c *C) { } func (s *S) TestViewWithCollation(c *C) { - // This test is currently failing because of a bug in mongodb. A ticket describing - // the issue is available here: https://jira.mongodb.org/browse/SERVER-31049 - // TODO remove this line when SERVER-31049 is fixed - c.Skip("Fails because of a MongoDB bug as of version 3.4.9, cf https://jira.mongodb.org/browse/SERVER-31049") - - if !s.versionAtLeast(3, 4) { + // SERVER-31049 is fixed in 3.4.10 + if !s.versionAtLeast(3, 4, 10) { c.Skip("depends on mongodb 3.4+") } // CreateView has to be run against mongos - session, err := mgo.Dial("localhost:40201") + session, err := mgo.Dial("localhost:40201" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1442,7 +1416,7 @@ func (s *S) TestViewWithCollation(c *C) { } func (s *S) TestCountQuery(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1460,7 +1434,7 @@ func (s *S) TestCountQuery(c *C) { } func (s *S) TestCountQuerySorted(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1478,7 +1452,7 @@ func (s *S) TestCountQuerySorted(c *C) { } func (s *S) TestCountSkipLimit(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1504,7 +1478,7 @@ func (s *S) TestCountMaxTimeMS(c *C) { c.Skip("SetMaxTime only supported in 2.6+") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1527,7 +1501,7 @@ func (s *S) TestCountHint(c *C) { c.Skip("Not implemented until mongo 2.5.5 https://jira.mongodb.org/browse/SERVER-2677") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1543,7 +1517,7 @@ func (s *S) TestCountHint(c *C) { } func (s *S) TestQueryExplain(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1578,7 +1552,7 @@ func (s *S) TestQueryExplain(c *C) { } func (s *S) TestQuerySetMaxScan(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() coll := session.DB("mydb").C("mycoll") @@ -1601,7 +1575,7 @@ func (s *S) TestQuerySetMaxTime(c *C) { c.Skip("SetMaxTime only supported in 2.6+") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() coll := session.DB("mydb").C("mycoll") @@ -1620,7 +1594,7 @@ func (s *S) TestQuerySetMaxTime(c *C) { } func (s *S) TestQueryHint(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1643,14 +1617,14 @@ func (s *S) TestQueryHint(c *C) { } func (s *S) TestQueryComment(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() db := session.DB("mydb") coll := db.C("mycoll") - err = db.Run(bson.M{"profile": 2}, nil) + err = db.Run(bson.D{{Name: "profile", Value: 2}}, nil) c.Assert(err, IsNil) ns := []int{40, 41, 42} @@ -1672,16 +1646,24 @@ func (s *S) TestQueryComment(c *C) { commentField := "query.$comment" nField := "query.$query.n" if s.versionAtLeast(3, 2) { - commentField = "query.comment" - nField = "query.filter.n" + if s.versionAtLeast(3, 6) { + commentField = "command.comment" + nField = "command.filter.n" + } else { + commentField = "query.comment" + nField = "query.filter.n" + } } n, err := session.DB("mydb").C("system.profile").Find(bson.M{nField: 41, commentField: "some comment"}).Count() c.Assert(err, IsNil) c.Assert(n, Equals, 1) + + err = db.Run(bson.D{{Name: "profile", Value: 0}}, nil) + c.Assert(err, IsNil) } func (s *S) TestFindOneNotFound(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1695,7 +1677,7 @@ func (s *S) TestFindOneNotFound(c *C) { } func (s *S) TestFindIterNotFound(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1709,7 +1691,7 @@ func (s *S) TestFindIterNotFound(c *C) { } func (s *S) TestFindNil(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1725,7 +1707,7 @@ func (s *S) TestFindNil(c *C) { } func (s *S) TestFindId(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1743,7 +1725,7 @@ func (s *S) TestFindId(c *C) { } func (s *S) TestFindIterAll(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1789,7 +1771,7 @@ func (s *S) TestFindIterAll(c *C) { } func (s *S) TestFindIterTwiceWithSameQuery(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1815,7 +1797,7 @@ func (s *S) TestFindIterTwiceWithSameQuery(c *C) { } func (s *S) TestFindIterWithoutResults(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1832,7 +1814,7 @@ func (s *S) TestFindIterWithoutResults(c *C) { } func (s *S) TestFindIterLimit(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1884,7 +1866,7 @@ func (s *S) TestResumeIter(c *C) { } const numDocuments = 10 - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) session.SetBatch(4) c.Assert(err, IsNil) defer session.Close() @@ -1946,7 +1928,7 @@ func (s *S) TestFindIterCursorTimeout(c *C) { if !*cursorTimeout { c.Skip("-cursor-timeout") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -1988,7 +1970,7 @@ func (s *S) TestFindIterCursorNoTimeout(c *C) { if !*cursorTimeout { c.Skip("-cursor-timeout") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -2039,7 +2021,7 @@ func (s *S) TestTooManyItemsLimitBug(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(runtime.NumCPU())) @@ -2075,7 +2057,7 @@ func (s *S) TestBatchSizeZeroGetMore(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(runtime.NumCPU())) @@ -2121,7 +2103,7 @@ func (s *S) TestFindIterLimitWithMore(c *C) { if s.versionAtLeast(3, 4) { c.Skip("fail on 3.4+") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -2184,7 +2166,7 @@ func (s *S) TestFindIterLimitWithMore(c *C) { } func (s *S) TestFindIterLimitWithBatch(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -2243,7 +2225,7 @@ func (s *S) TestFindIterLimitWithBatch(c *C) { } func (s *S) TestFindIterSortWithBatch(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -2310,7 +2292,7 @@ func (s *S) TestFindTailTimeoutWithSleep(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -2404,7 +2386,7 @@ func (s *S) TestFindTailTimeoutWithSleep(c *C) { // Test tailable cursors in a situation where Next never gets to sleep once // to respect the timeout requested on Tail. func (s *S) TestFindTailTimeoutNoSleep(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -2489,7 +2471,7 @@ func (s *S) TestFindTailNoTimeout(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -2574,7 +2556,7 @@ func (s *S) TestFindTailNoTimeout(c *C) { } func (s *S) TestIterNextResetsResult(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -2644,7 +2626,7 @@ func (s *S) TestIterNextResetsResult(c *C) { } func (s *S) TestFindForOnIter(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -2696,7 +2678,7 @@ func (s *S) TestFindForOnIter(c *C) { } func (s *S) TestFindFor(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -2748,7 +2730,7 @@ func (s *S) TestFindFor(c *C) { } func (s *S) TestFindForStopOnError(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -2775,7 +2757,7 @@ func (s *S) TestFindForStopOnError(c *C) { } func (s *S) TestFindForResetsResult(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -2845,11 +2827,8 @@ func (s *S) TestFindForResetsResult(c *C) { } func (s *S) TestFindIterSnapshot(c *C) { - if s.versionAtLeast(3, 2) { - c.Skip("Broken in 3.2: https://jira.mongodb.org/browse/SERVER-21403") - } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -2893,7 +2872,7 @@ func (s *S) TestFindIterSnapshot(c *C) { } func (s *S) TestSort(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -2928,7 +2907,7 @@ func (s *S) TestSort(c *C) { } func (s *S) TestSortWithBadArgs(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -2944,7 +2923,7 @@ func (s *S) TestSortWithBadArgs(c *C) { } func (s *S) TestSortScoreText(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -3004,7 +2983,7 @@ func (s *S) TestSortScoreText(c *C) { } func (s *S) TestPrefetching(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -3093,7 +3072,8 @@ func (s *S) TestPrefetching(c *C) { } func (s *S) TestSafeSetting(c *C) { - session, err := mgo.Dial("localhost:40001") + + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -3206,7 +3186,7 @@ func (s *S) TestSafeSetting(c *C) { } func (s *S) TestSafeInsert(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -3221,16 +3201,16 @@ func (s *S) TestSafeInsert(c *C) { // Session should be safe by default, so inserting it again must fail. err = coll.Insert(M{"_id": 1}) c.Assert(err, ErrorMatches, ".*E11000 duplicate.*") - c.Assert(err.(*mgo.LastError).Code, Equals, 11000) + if lerr, ok := err.(*mgo.LastError); ok { + c.Assert(lerr.Code, Equals, 11000) + } else { + c.Assert(err.(*mgo.QueryError).Code, Equals, 11000) + } - // It must have sent two operations (INSERT_OP + getLastError QUERY_OP) + // It must have sent one operation stats := mgo.GetStats() - if s.versionAtLeast(2, 6) { - c.Assert(stats.SentOps, Equals, 1) - } else { - c.Assert(stats.SentOps, Equals, 2) - } + c.Assert(stats.SentOps, Equals, 1) mgo.ResetStats() @@ -3245,7 +3225,8 @@ func (s *S) TestSafeInsert(c *C) { } func (s *S) TestSafeParameters(c *C) { - session, err := mgo.Dial("localhost:40011") + + session, err := mgo.Dial("localhost:40011" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -3262,7 +3243,7 @@ func (s *S) TestSafeParameters(c *C) { } func (s *S) TestQueryErrorOne(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -3282,7 +3263,7 @@ func (s *S) TestQueryErrorOne(c *C) { } func (s *S) TestQueryErrorNext(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -3486,7 +3467,7 @@ func getIndex34(session *mgo.Session, db, collection, name string) M { } func (s *S) TestEnsureIndex(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -3528,7 +3509,7 @@ func (s *S) TestEnsureIndex(c *C) { // db.runCommand({"listIndexes": }) // // and iterate over the returned cursor. - if s.versionAtLeast(3, 4) { + if s.versionAtLeast(3, 2, 17) { c.Assert(getIndex34(session, "mydb", "mycoll", test.expected["name"].(string)), DeepEquals, test.expected) } else { idxs := session.DB("mydb").C("system.indexes") @@ -3590,7 +3571,7 @@ func (s *S) TestEnsureIndex(c *C) { } func (s *S) TestEnsureIndexWithBadInfo(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -3604,7 +3585,7 @@ func (s *S) TestEnsureIndexWithBadInfo(c *C) { } func (s *S) TestEnsureIndexWithUnsafeSession(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -3629,7 +3610,7 @@ func (s *S) TestEnsureIndexWithUnsafeSession(c *C) { } func (s *S) TestEnsureIndexKey(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -3638,7 +3619,7 @@ func (s *S) TestEnsureIndexKey(c *C) { err = coll.EnsureIndexKey("a") c.Assert(err, IsNil) - if s.versionAtLeast(3, 4) { + if s.versionAtLeast(3, 2, 17) { expected := M{ "name": "a_1", "key": M{"a": 1}, @@ -3688,7 +3669,7 @@ func (s *S) TestEnsureIndexKey(c *C) { } func (s *S) TestEnsureIndexDropIndex(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -3703,7 +3684,7 @@ func (s *S) TestEnsureIndexDropIndex(c *C) { err = coll.DropIndex("-b") c.Assert(err, IsNil) - if s.versionAtLeast(3, 4) { + if s.versionAtLeast(3, 2, 17) { // system.indexes is deprecated since 3.0, use // db.runCommand({"listIndexes": }) // instead @@ -3744,7 +3725,7 @@ func (s *S) TestEnsureIndexDropIndex(c *C) { } func (s *S) TestEnsureIndexDropIndexName(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -3758,7 +3739,7 @@ func (s *S) TestEnsureIndexDropIndexName(c *C) { err = coll.DropIndexName("a") c.Assert(err, IsNil) - if s.versionAtLeast(3, 4) { + if s.versionAtLeast(3, 2, 17) { // system.indexes is deprecated since 3.0, use // db.runCommand({"listIndexes": }) // instead @@ -3799,7 +3780,7 @@ func (s *S) TestEnsureIndexDropIndexName(c *C) { } func (s *S) TestEnsureIndexDropAllIndexes(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -3814,7 +3795,7 @@ func (s *S) TestEnsureIndexDropAllIndexes(c *C) { err = coll.DropAllIndexes() c.Assert(err, IsNil) - if s.versionAtLeast(3, 4) { + if s.versionAtLeast(3, 2, 17) { // system.indexes is deprecated since 3.0, use // db.runCommand({"listIndexes": }) // instead @@ -3832,7 +3813,7 @@ func (s *S) TestEnsureIndexDropAllIndexes(c *C) { } func (s *S) TestEnsureIndexCaching(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -3873,7 +3854,7 @@ func (s *S) TestEnsureIndexCaching(c *C) { } func (s *S) TestEnsureIndexGetIndexes(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -3910,7 +3891,7 @@ func (s *S) TestEnsureIndexGetIndexes(c *C) { } func (s *S) TestEnsureIndexNameCaching(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -3954,7 +3935,7 @@ func (s *S) TestEnsureIndexNameCaching(c *C) { } func (s *S) TestEnsureIndexEvalGetIndexes(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -3991,7 +3972,7 @@ func (s *S) TestEnsureIndexEvalGetIndexes(c *C) { var testTTL = flag.Bool("test-ttl", false, "test TTL collections (may take 1 minute)") func (s *S) TestEnsureIndexExpireAfter(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4039,7 +4020,7 @@ func (s *S) TestEnsureIndexExpireAfter(c *C) { } func (s *S) TestDistinct(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4057,7 +4038,7 @@ func (s *S) TestDistinct(c *C) { } func (s *S) TestMapReduce(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4092,7 +4073,7 @@ func (s *S) TestMapReduce(c *C) { } func (s *S) TestMapReduceFinalize(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4123,7 +4104,7 @@ func (s *S) TestMapReduceFinalize(c *C) { } func (s *S) TestMapReduceToCollection(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4163,7 +4144,7 @@ func (s *S) TestMapReduceToCollection(c *C) { } func (s *S) TestMapReduceToOtherDb(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4203,7 +4184,7 @@ func (s *S) TestMapReduceToOtherDb(c *C) { } func (s *S) TestMapReduceOutOfOrder(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4226,7 +4207,7 @@ func (s *S) TestMapReduceOutOfOrder(c *C) { } func (s *S) TestMapReduceScope(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4248,7 +4229,7 @@ func (s *S) TestMapReduceScope(c *C) { } func (s *S) TestMapReduceVerbose(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4271,7 +4252,7 @@ func (s *S) TestMapReduceVerbose(c *C) { } func (s *S) TestMapReduceLimit(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4293,7 +4274,7 @@ func (s *S) TestMapReduceLimit(c *C) { } func (s *S) TestBuildInfo(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4334,7 +4315,7 @@ func (s *S) TestBuildInfo(c *C) { } func (s *S) TestZeroTimeRoundtrip(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4352,7 +4333,7 @@ func (s *S) TestZeroTimeRoundtrip(c *C) { } func (s *S) TestFsyncLock(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4380,7 +4361,7 @@ func (s *S) TestFsyncLock(c *C) { } func (s *S) TestFsync(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4395,11 +4376,11 @@ func (s *S) TestRepairCursor(c *C) { if !s.versionAtLeast(2, 7) { c.Skip("RepairCursor only works on 2.7+") } - if s.versionAtLeast(3, 4) { - c.Skip("fail on 3.4+") + if s.versionAtLeast(3, 2, 17) { + c.Skip("fail on 3.2.17+") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() session.SetBatch(2) @@ -4446,7 +4427,7 @@ func (s *S) TestPipeIter(c *C) { c.Skip("Pipe only works on 2.1+") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4482,7 +4463,7 @@ func (s *S) TestPipeAll(c *C) { c.Skip("Pipe only works on 2.1+") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4507,7 +4488,7 @@ func (s *S) TestPipeOne(c *C) { c.Skip("Pipe only works on 2.1+") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4532,7 +4513,7 @@ func (s *S) TestPipeExplain(c *C) { c.Skip("Pipe only works on 2.1+") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4549,7 +4530,7 @@ func (s *S) TestPipeExplain(c *C) { } func (s *S) TestBatch1Bug(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4572,7 +4553,7 @@ func (s *S) TestBatch1Bug(c *C) { } func (s *S) TestInterfaceIterBug(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4595,7 +4576,7 @@ func (s *S) TestInterfaceIterBug(c *C) { } func (s *S) TestFindIterCloseKillsCursor(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4616,7 +4597,7 @@ func (s *S) TestFindIterCloseKillsCursor(c *C) { } func (s *S) TestFindIterDoneWithBatches(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4645,7 +4626,7 @@ func (s *S) TestFindIterDoneWithBatches(c *C) { } func (s *S) TestFindIterDoneErr(c *C) { - session, err := mgo.Dial("localhost:40002") + session, err := mgo.Dial("localhost:40002" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4660,7 +4641,7 @@ func (s *S) TestFindIterDoneErr(c *C) { } func (s *S) TestFindIterDoneNotFound(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4674,7 +4655,7 @@ func (s *S) TestFindIterDoneNotFound(c *C) { } func (s *S) TestLogReplay(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4695,7 +4676,7 @@ func (s *S) TestLogReplay(c *C) { } func (s *S) TestSetCursorTimeout(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4715,7 +4696,7 @@ func (s *S) TestSetCursorTimeout(c *C) { } func (s *S) TestNewIterNoServer(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4737,7 +4718,7 @@ func (s *S) TestNewIterNoServer(c *C) { } func (s *S) TestNewIterNoServerPresetErr(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4762,7 +4743,7 @@ func (s *S) TestBypassValidation(c *C) { if !s.versionAtLeast(3, 2) { c.Skip("validation supported on 3.2+") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4832,7 +4813,7 @@ func (s *S) TestCollationQueries(c *C) { if !s.versionAtLeast(3, 3, 12) { c.Skip("collations being released with 3.4") } - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4874,7 +4855,7 @@ func (s *S) TestCollationQueries(c *C) { // Some benchmarks that require a running database. func (s *S) BenchmarkFindIterRaw(c *C) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) c.Assert(err, IsNil) defer session.Close() @@ -4906,7 +4887,7 @@ func (s *S) BenchmarkFindIterRaw(c *C) { } func BenchmarkInsertSingle(b *testing.B) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) if err != nil { b.Fatal(err) } @@ -4926,7 +4907,7 @@ func BenchmarkInsertSingle(b *testing.B) { } func BenchmarkInsertMultiple(b *testing.B) { - session, err := mgo.Dial("localhost:40001") + session, err := mgo.Dial("localhost:40001" + expFeaturesString) if err != nil { b.Fatal(err) } diff --git a/socket.go b/socket.go index f6158189c..f739baf9c 100644 --- a/socket.go +++ b/socket.go @@ -29,6 +29,7 @@ package mgo import ( "errors" "fmt" + "io" "net" "sync" "time" @@ -38,22 +39,24 @@ import ( type replyFunc func(err error, reply *replyOp, docNum int, docData []byte) +type opMsgReplyFunc func(reply *msgOp, err error) + type mongoSocket struct { sync.Mutex - server *mongoServer // nil when cached - conn net.Conn - timeout time.Duration - addr string // For debugging only. - nextRequestId uint32 - replyFuncs map[uint32]replyFunc - references int - creds []Credential - logout []Credential - cachedNonce string - gotNonce sync.Cond - dead error - serverInfo *mongoServerInfo - closeAfterIdle bool + server *mongoServer // nil when cached + conn net.Conn + timeout time.Duration + addr string // For debugging only. + nextRequestId uint32 + replyFuncs map[uint32]replyFunc + opMsgReplyFuncs map[uint32]opMsgReplyFunc + references int + creds []Credential + cachedNonce string + gotNonce sync.Cond + dead error + serverInfo *mongoServerInfo + closeAfterIdle bool } type queryOpFlags uint32 @@ -65,6 +68,29 @@ const ( flagLogReplay flagNoCursorTimeout flagAwaitData + // section type, as defined here: + // https://docs.mongodb.com/master/reference/mongodb-wire-protocol/#sections + msgPayload0 = uint8(0) + msgPayload1 = uint8(1) + // all possible opCodes, as defined here: + // https://docs.mongodb.com/manual/reference/mongodb-wire-protocol/#request-opcodes + opInvalid = 0 + opReply = 1 + dbMsg = 1000 + dbUpdate = 2001 + dbInsert = 2002 + dbQuery = 2004 + dbGetMore = 2005 + dbDelete = 2006 + dbKillCursors = 2007 + dbCommand = 2010 + dbCommandReply = 2011 + dbCompressed = 2012 + dbMessage = 2013 + // opMsg flags + opMsgFlagChecksumPresent = 1 + opMsgFlagMoreToCome = (1 << 1) + opMsgFlagExhaustAllowed = (1 << 16) ) type queryOp struct { @@ -174,6 +200,29 @@ type killCursorsOp struct { cursorIds []int64 } +type msgSection struct { + payloadType uint8 + data interface{} +} + +// op_msg is introduced in mongodb 3.6, see +// https://docs.mongodb.com/master/reference/mongodb-wire-protocol/#op-msg +// for details +type msgOp struct { + flags uint32 + sections []msgSection + checksum uint32 +} + +// PayloadType1 is a container for the OP_MSG payload data of type 1. +// There is no definition of the type 0 payload because that is simply a +// bson document. +type payloadType1 struct { + size int32 + identifier string + docs []interface{} +} + type requestInfo struct { bufferPos int replyFunc replyFunc @@ -181,10 +230,11 @@ type requestInfo struct { func newSocket(server *mongoServer, conn net.Conn, timeout time.Duration) *mongoSocket { socket := &mongoSocket{ - conn: conn, - addr: server.Addr, - server: server, - replyFuncs: make(map[uint32]replyFunc), + conn: conn, + addr: server.Addr, + server: server, + replyFuncs: make(map[uint32]replyFunc), + opMsgReplyFuncs: make(map[uint32]opMsgReplyFunc), } socket.gotNonce.L = &socket.Mutex if err := socket.InitialAcquire(server.Info(), timeout); err != nil { @@ -352,6 +402,8 @@ func (socket *mongoSocket) kill(err error, abend bool) { stats.socketsAlive(-1) replyFuncs := socket.replyFuncs socket.replyFuncs = make(map[uint32]replyFunc) + opMsgReplyFuncs := socket.opMsgReplyFuncs + socket.opMsgReplyFuncs = make(map[uint32]opMsgReplyFunc) server := socket.server socket.server = nil socket.gotNonce.Broadcast() @@ -360,6 +412,10 @@ func (socket *mongoSocket) kill(err error, abend bool) { logf("Socket %p to %s: notifying replyFunc of closed socket: %s", socket, socket.addr, err.Error()) replyFunc(err, nil, -1, nil) } + for _, opMsgReplyFunc := range opMsgReplyFuncs { + logf("Socket %p to %s: notifying replyFunc of closed socket: %s", socket, socket.addr, err.Error()) + opMsgReplyFunc(nil, err) + } if abend { server.AbendSocket(socket) } @@ -403,14 +459,8 @@ var bytesBufferPool = sync.Pool{ func (socket *mongoSocket) Query(ops ...interface{}) (err error) { - if lops := socket.flushLogout(); len(lops) > 0 { - ops = append(lops, ops...) - } - - buf := bytesBufferPool.Get().([]byte) - defer func() { - bytesBufferPool.Put(buf[:0]) - }() + buf := getSizedBuffer(0) + defer bytesBufferPool.Put(buf) // Serialize operations synchronously to avoid interrupting // other goroutines while we can't really be sending data. @@ -431,7 +481,7 @@ func (socket *mongoSocket) Query(ops ...interface{}) (err error) { switch op := op.(type) { case *updateOp: - buf = addHeader(buf, 2001) + buf = addHeader(buf, dbUpdate) buf = addInt32(buf, 0) // Reserved buf = addCString(buf, op.Collection) buf = addInt32(buf, int32(op.Flags)) @@ -447,7 +497,7 @@ func (socket *mongoSocket) Query(ops ...interface{}) (err error) { } case *insertOp: - buf = addHeader(buf, 2002) + buf = addHeader(buf, dbInsert) buf = addInt32(buf, int32(op.flags)) buf = addCString(buf, op.collection) for _, doc := range op.documents { @@ -459,7 +509,7 @@ func (socket *mongoSocket) Query(ops ...interface{}) (err error) { } case *queryOp: - buf = addHeader(buf, 2004) + buf = addHeader(buf, dbQuery) buf = addInt32(buf, int32(op.flags)) buf = addCString(buf, op.collection) buf = addInt32(buf, op.skip) @@ -477,7 +527,7 @@ func (socket *mongoSocket) Query(ops ...interface{}) (err error) { replyFunc = op.replyFunc case *getMoreOp: - buf = addHeader(buf, 2005) + buf = addHeader(buf, dbGetMore) buf = addInt32(buf, 0) // Reserved buf = addCString(buf, op.collection) buf = addInt32(buf, op.limit) @@ -485,7 +535,7 @@ func (socket *mongoSocket) Query(ops ...interface{}) (err error) { replyFunc = op.replyFunc case *deleteOp: - buf = addHeader(buf, 2006) + buf = addHeader(buf, dbDelete) buf = addInt32(buf, 0) // Reserved buf = addCString(buf, op.Collection) buf = addInt32(buf, int32(op.Flags)) @@ -496,7 +546,7 @@ func (socket *mongoSocket) Query(ops ...interface{}) (err error) { } case *killCursorsOp: - buf = addHeader(buf, 2007) + buf = addHeader(buf, dbKillCursors) buf = addInt32(buf, 0) // Reserved buf = addInt32(buf, int32(len(op.cursorIds))) for _, cursorId := range op.cursorIds { @@ -561,123 +611,245 @@ func (socket *mongoSocket) Query(ops ...interface{}) (err error) { return err } -func fill(r net.Conn, b []byte) error { - l := len(b) - n, err := r.Read(b) - for n != l && err == nil { - var ni int - ni, err = r.Read(b[n:]) - n += ni +// sendMessage send data to the database using the OP_MSG wire protocol +// introduced in MongoDB 3.6 (require maxWireVersion >= 6) +func (socket *mongoSocket) sendMessage(op *msgOp) (writeCmdResult, error) { + var wr writeCmdResult + var err error + + buf := getSizedBuffer(0) + defer bytesBufferPool.Put(buf) + + buf = addHeader(buf, dbMessage) + buf = addInt32(buf, int32(op.flags)) + + for _, section := range op.sections { + buf, err = addSection(buf, section) + if err != nil { + return wr, err + } } - return err + + if len(buf) > socket.ServerInfo().MaxMessageSizeBytes { + return wr, fmt.Errorf("message length to long, should be < %v, but was %v", socket.ServerInfo().MaxMessageSizeBytes, len(buf)) + } + // set the total message size + setInt32(buf, 0, int32(len(buf))) + + var wait sync.Mutex + var reply msgOp + var responseError error + var wcr writeCmdResult + // if no response expected, ie op.flags&opMsgFlagMoreToCome == 1, + // request should have id 0 + var requestID uint32 + // if moreToCome flag is set, we don't want to know the outcome of the message. + // There is no response to a request where moreToCome has been set. + expectReply := (op.flags & opMsgFlagMoreToCome) == 0 + + socket.Lock() + if socket.dead != nil { + dead := socket.dead + socket.Unlock() + debugf("Socket %p to %s: failing query, already closed: %s", socket, socket.addr, socket.dead.Error()) + return wr, dead + } + if expectReply { + // Reserve id 0 for requests which should have no responses. + again: + requestID = socket.nextRequestId + 1 + socket.nextRequestId++ + if requestID == 0 { + goto again + } + wait.Lock() + socket.opMsgReplyFuncs[requestID] = func(msg *msgOp, err error) { + reply = *msg + responseError = err + wait.Unlock() + } + } + socket.Unlock() + + setInt32(buf, 4, int32(requestID)) + stats.sentOps(1) + + socket.updateDeadline(writeDeadline) + _, err = socket.conn.Write(buf) + + if expectReply { + socket.updateDeadline(readDeadline) + wait.Lock() + + if responseError != nil { + return wcr, responseError + } + // for the moment, OP_MSG responses return a body section only, + // cf https://github.com/mongodb/specifications/blob/master/source/message/OP_MSG.rst : + // + // "Similarly, certain commands will reply to messages using this technique when possible + // to avoid the overhead of BSON Arrays. Drivers will be required to allow all command + // replies to use this technique. Drivers will be required to handle Payload Type 1." + // + // so we only return the first section of the response (ie the body) + wcr = reply.sections[0].data.(writeCmdResult) + } + + return wcr, err +} + +// get a slice of byte of `size` length from the pool +func getSizedBuffer(size int) []byte { + b := bytesBufferPool.Get().([]byte) + if len(b) < size { + for i := len(b); i < size; i++ { + b = append(b, byte(0)) + } + return b + } + return b[0:size] } // Estimated minimum cost per socket: 1 goroutine + memory for the largest // document ever seen. func (socket *mongoSocket) readLoop() { - p := make([]byte, 36) // 16 from header + 20 from OP_REPLY fixed fields + header := make([]byte, 16) // 16 bytes for header + p := make([]byte, 20) // 20 bytes for fixed fields of OP_REPLY s := make([]byte, 4) - conn := socket.conn // No locking, conn never changes. + var r io.Reader = socket.conn // No locking, conn never changes. for { - err := fill(conn, p) + _, err := io.ReadFull(r, header) if err != nil { socket.kill(err, true) return } - totalLen := getInt32(p, 0) - responseTo := getInt32(p, 8) - opCode := getInt32(p, 12) + totalLen := getInt32(header, 0) + responseTo := getInt32(header, 8) + opCode := getInt32(header, 12) // Don't use socket.server.Addr here. socket is not // locked and socket.server may go away. debugf("Socket %p to %s: got reply (%d bytes)", socket, socket.addr, totalLen) + stats.receivedOps(1) - _ = totalLen - - if opCode != 1 { - socket.kill(errors.New("opcode != 1, corrupted data?"), true) - return - } - - reply := replyOp{ - flags: uint32(getInt32(p, 16)), - cursorId: getInt64(p, 20), - firstDoc: getInt32(p, 28), - replyDocs: getInt32(p, 32), - } - - stats.receivedOps(+1) - stats.receivedDocs(int(reply.replyDocs)) - - socket.Lock() - replyFunc, ok := socket.replyFuncs[uint32(responseTo)] - if ok { - delete(socket.replyFuncs, uint32(responseTo)) - } - socket.Unlock() + switch opCode { + case opReply: + _, err := io.ReadFull(r, p) + if err != nil { + socket.kill(err, true) + return + } + reply := replyOp{ + flags: uint32(getInt32(p, 0)), + cursorId: getInt64(p, 4), + firstDoc: getInt32(p, 12), + replyDocs: getInt32(p, 16), + } + stats.receivedDocs(int(reply.replyDocs)) - if replyFunc != nil && reply.replyDocs == 0 { - replyFunc(nil, &reply, -1, nil) - } else { - for i := 0; i != int(reply.replyDocs); i++ { - err := fill(conn, s) - if err != nil { - if replyFunc != nil { - replyFunc(err, nil, -1, nil) + socket.Lock() + replyFunc, ok := socket.replyFuncs[uint32(responseTo)] + if ok { + delete(socket.replyFuncs, uint32(responseTo)) + } + socket.Unlock() + + if replyFunc != nil && reply.replyDocs == 0 { + replyFunc(nil, &reply, -1, nil) + } else { + for i := 0; i != int(reply.replyDocs); i++ { + _, err := io.ReadFull(r, s) + if err != nil { + if replyFunc != nil { + replyFunc(err, nil, -1, nil) + } + socket.kill(err, true) + return + } + b := getSizedBuffer(int(getInt32(s, 0))) + defer bytesBufferPool.Put(b) + + copy(b[0:4], s) + + _, err = io.ReadFull(r, b[4:]) + if err != nil { + if replyFunc != nil { + replyFunc(err, nil, -1, nil) + } + socket.kill(err, true) + return } - socket.kill(err, true) - return - } - - b := make([]byte, int(getInt32(s, 0))) - // copy(b, s) in an efficient way. - b[0] = s[0] - b[1] = s[1] - b[2] = s[2] - b[3] = s[3] + if globalDebug && globalLogger != nil { + m := bson.M{} + if err := bson.Unmarshal(b, m); err == nil { + debugf("Socket %p to %s: received document: %#v", socket, socket.addr, m) + } + } - err = fill(conn, b[4:]) - if err != nil { if replyFunc != nil { - replyFunc(err, nil, -1, nil) + replyFunc(nil, &reply, i, b) } - socket.kill(err, true) - return + // XXX Do bound checking against totalLen. } + } - if globalDebug && globalLogger != nil { - m := bson.M{} - if err := bson.Unmarshal(b, m); err == nil { - debugf("Socket %p to %s: received document: %#v", socket, socket.addr, m) - } - } + socket.Lock() + if len(socket.replyFuncs) == 0 { + // Nothing else to read for now. Disable deadline. + socket.conn.SetReadDeadline(time.Time{}) + } else { + socket.updateDeadline(readDeadline) + } + socket.Unlock() - if replyFunc != nil { - replyFunc(nil, &reply, i, b) - } + case dbMessage: + body := getSizedBuffer(int(totalLen) - 16) + defer bytesBufferPool.Put(body) + _, err := io.ReadFull(r, body) + if err != nil { + socket.kill(err, true) + return + } - // XXX Do bound checking against totalLen. + sections, err := getSections(body[4:]) + if err != nil { + socket.kill(err, true) + return + } + // TODO check CRC-32 checksum if checksum byte is set + reply := &msgOp{ + flags: uint32(getInt32(body, 0)), + sections: sections, } - } - socket.Lock() - if len(socket.replyFuncs) == 0 { - // Nothing else to read for now. Disable deadline. + // TODO update this when msgPayload1 section is implemented in MongoDB + stats.receivedDocs(1) + socket.Lock() + opMsgReplyFunc, ok := socket.opMsgReplyFuncs[uint32(responseTo)] + if ok { + delete(socket.opMsgReplyFuncs, uint32(responseTo)) + } + socket.Unlock() + + if opMsgReplyFunc != nil { + opMsgReplyFunc(reply, err) + } else { + socket.kill(fmt.Errorf("couldn't handle response properly"), true) + return + } socket.conn.SetReadDeadline(time.Time{}) - } else { - socket.updateDeadline(readDeadline) + default: + socket.kill(errors.New("opcode != 1 && opcode != 2013, corrupted data?"), true) + return } - socket.Unlock() - - // XXX Do bound checking against totalLen. } } var emptyHeader = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} -func addHeader(b []byte, opcode int) []byte { +func addHeader(b []byte, opcode int32) []byte { i := len(b) b = append(b, emptyHeader...) // Enough for current opcodes. @@ -701,6 +873,35 @@ func addCString(b []byte, s string) []byte { return b } +// Marshal a section and add it to the provided buffer +// https://docs.mongodb.com/master/reference/mongodb-wire-protocol/#sections +func addSection(b []byte, s msgSection) ([]byte, error) { + var err error + b = append(b, s.payloadType) + switch s.payloadType { + case msgPayload0: + b, err = addBSON(b, s.data) + if err != nil { + return b, err + } + case msgPayload1: + pos := len(b) + b = addInt32(b, 0) + s1 := s.data.(payloadType1) + b = addCString(b, s1.identifier) + for _, doc := range s1.docs { + b, err = bson.MarshalBuffer(doc, b) + if err != nil { + return b, err + } + } + setInt32(b, pos, int32(len(b)-pos)) + default: + return b, fmt.Errorf("invalid section kind in op_msg: %v", s.payloadType) + } + return b, nil +} + func addBSON(b []byte, doc interface{}) ([]byte, error) { if doc == nil { return append(b, 5, 0, 0, 0, 0), nil @@ -736,3 +937,36 @@ func getInt64(b []byte, pos int) int64 { (int64(b[pos+6]) << 48) | (int64(b[pos+7]) << 56) } + +// UnMarshal an array of bytes into a section +// https://docs.mongodb.com/master/reference/mongodb-wire-protocol/#sections +func getSections(b []byte) ([]msgSection, error) { + var sections []msgSection + pos := 0 + for pos != len(b) { + sectionLength := int(getInt32(b, pos+1)) + // first byte is section type + switch b[pos] { + case msgPayload0: + var result writeCmdResult + err := bson.Unmarshal(b[pos+1:pos+sectionLength+1], &result) + if err != nil { + return nil, err + } + sections = append(sections, msgSection{ + payloadType: b[pos], + data: result, + }) + case msgPayload1: + // not implemented yet + // + // b[0:4] size + // b[4:?] docSeqID + // b[?:len(b)] documentSequence + default: + return nil, fmt.Errorf("invalid section type: %v", b[0]) + } + pos += sectionLength + 1 + } + return sections, nil +} diff --git a/suite_test.go b/suite_test.go index 624d5a543..e4a149569 100644 --- a/suite_test.go +++ b/suite_test.go @@ -45,6 +45,8 @@ import ( var fast = flag.Bool("fast", false, "Skip slow tests") +var expFeaturesString = "?experimental=opmsg" + type M bson.M type cLogger C From a104bfb56b2dd52d66fb6c0cf2f83c712751f07a Mon Sep 17 00:00:00 2001 From: Bachue Zhou Date: Thu, 28 Dec 2017 01:11:29 +0800 Subject: [PATCH 27/38] Recover from persistent "i/o timeout" or "Closed explicitly" pool errors (#69) --- session.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/session.go b/session.go index cdce36919..e31323f92 100644 --- a/session.go +++ b/session.go @@ -4800,13 +4800,13 @@ func (s *Session) acquireSocket(slaveOk bool) (*mongoSocket, error) { s.m.RLock() // If there is a slave socket reserved and its use is acceptable, take it as long // as there isn't a master socket which would be preferred by the read preference mode. - if s.slaveSocket != nil && s.slaveOk && slaveOk && (s.masterSocket == nil || s.consistency != PrimaryPreferred && s.consistency != Monotonic) { + if s.slaveSocket != nil && s.slaveSocket.dead == nil && s.slaveOk && slaveOk && (s.masterSocket == nil || s.consistency != PrimaryPreferred && s.consistency != Monotonic) { socket := s.slaveSocket socket.Acquire() s.m.RUnlock() return socket, nil } - if s.masterSocket != nil { + if s.masterSocket != nil && s.masterSocket.dead == nil { socket := s.masterSocket socket.Acquire() s.m.RUnlock() @@ -4820,12 +4820,20 @@ func (s *Session) acquireSocket(slaveOk bool) (*mongoSocket, error) { defer s.m.Unlock() if s.slaveSocket != nil && s.slaveOk && slaveOk && (s.masterSocket == nil || s.consistency != PrimaryPreferred && s.consistency != Monotonic) { - s.slaveSocket.Acquire() - return s.slaveSocket, nil + if s.slaveSocket.dead == nil { + s.slaveSocket.Acquire() + return s.slaveSocket, nil + } else { + s.unsetSocket() + } } if s.masterSocket != nil { - s.masterSocket.Acquire() - return s.masterSocket, nil + if s.masterSocket.dead == nil { + s.masterSocket.Acquire() + return s.masterSocket, nil + } else { + s.unsetSocket() + } } // Still not good. We need a new socket. @@ -4876,9 +4884,11 @@ func (s *Session) setSocket(socket *mongoSocket) { // unsetSocket releases any slave and/or master sockets reserved. func (s *Session) unsetSocket() { if s.masterSocket != nil { + debugf("unset master socket from session %p", s) s.masterSocket.Release() } if s.slaveSocket != nil { + debugf("unset slave socket from session %p", s) s.slaveSocket.Release() } s.masterSocket = nil From f9be6c5b92fa69154d4d39eb77046f1521f45cc2 Mon Sep 17 00:00:00 2001 From: Dom Date: Tue, 9 Jan 2018 10:24:07 +0000 Subject: [PATCH 28/38] development: revert #61 (#73) * Revert "MongoDB 3.6: implement the new wire protocol (#61)" This reverts commit 90c056c16eb749ea476643a0aa62d53035ac983e. * test against 3.6 * update go and mongodb version - use last minor version for each major serie of mongodb - use travis 'go' param instead of eval "$(gimme $GO)" as it fails to install correctly last minor version ( 1.8.x notation for example) * test fixes on 3.2.17 also re-enable TestFindIterSnapshot as it was fixed a long time ago * fix X509 test fix TestAuthX509CredRDNConstruction test: need to create an user with {"username": subject} before trying to login * Fix auth test on 3.6-rc3 Make sure that "rs3/127.0.0.1/40031" is elected at primary. Create user before running 'addShard' command as it requires root access Also add a retry mechanism to make sure that shard are correctly added cf https://docs.mongodb.com/manual/tutorial/deploy-shard-cluster/ * update to 3.6.0 stable * tests: cherry pick missing 3.6+ support changes --- auth.go | 57 +++--- auth_test.go | 93 +++++----- bulk_test.go | 34 ++-- cluster.go | 32 ++-- cluster_test.go | 118 ++++++------- gridfs_test.go | 24 +-- server.go | 27 +-- session.go | 191 +------------------- session_test.go | 296 +++++++++++++++---------------- socket.go | 458 ++++++++++++------------------------------------ suite_test.go | 2 - 11 files changed, 448 insertions(+), 884 deletions(-) diff --git a/auth.go b/auth.go index ed3baad15..75d2ebc36 100644 --- a/auth.go +++ b/auth.go @@ -170,6 +170,12 @@ func (socket *mongoSocket) Login(cred Credential) error { return nil } } + if socket.dropLogout(cred) { + debugf("Socket %p to %s: login: db=%q user=%q (cached)", socket, socket.addr, cred.Source, cred.Username) + socket.creds = append(socket.creds, cred) + socket.Unlock() + return nil + } socket.Unlock() debugf("Socket %p to %s: login: db=%q user=%q", socket, socket.addr, cred.Source, cred.Username) @@ -406,50 +412,36 @@ func (socket *mongoSocket) Logout(db string) { cred, found := socket.dropAuth(db) if found { debugf("Socket %p to %s: logout: db=%q (flagged)", socket, socket.addr, db) - socket.Unlock() - err := socket.flushLogout(cred) - if err != nil { - debugf("fail to logout for cred %v; error: %v", cred, err) - } - } else { - socket.Unlock() + socket.logout = append(socket.logout, cred) } + socket.Unlock() } func (socket *mongoSocket) LogoutAll() { socket.Lock() if l := len(socket.creds); l > 0 { - credCopy := make([]Credential, l) - copy(credCopy, socket.creds) - socket.creds = socket.creds[0:0] - socket.Unlock() debugf("Socket %p to %s: logout all (flagged %d)", socket, socket.addr, l) - err := socket.flushLogout(credCopy...) - if err != nil { - debugf("fail to logout for cred %v, error: %v", credCopy, err) - } - } else { - socket.Unlock() + socket.logout = append(socket.logout, socket.creds...) + socket.creds = socket.creds[0:0] } + socket.Unlock() } -func (socket *mongoSocket) flushLogout(cred ...Credential) error { - if l := len(cred); l > 0 { +func (socket *mongoSocket) flushLogout() (ops []interface{}) { + socket.Lock() + if l := len(socket.logout); l > 0 { debugf("Socket %p to %s: logout all (flushing %d)", socket, socket.addr, l) - ops := make([]interface{}, l) for i := 0; i != l; i++ { op := queryOp{} op.query = &logoutCmd{1} - op.collection = cred[i].Source + ".$cmd" + op.collection = socket.logout[i].Source + ".$cmd" op.limit = -1 - ops[i] = &op - } - err := socket.Query(ops...) - if err != nil { - return fmt.Errorf("fail to logout: %v", err) + ops = append(ops, &op) } + socket.logout = socket.logout[0:0] } - return nil + socket.Unlock() + return } func (socket *mongoSocket) dropAuth(db string) (cred Credential, found bool) { @@ -462,3 +454,14 @@ func (socket *mongoSocket) dropAuth(db string) (cred Credential, found bool) { } return cred, false } + +func (socket *mongoSocket) dropLogout(cred Credential) (found bool) { + for i, sockCred := range socket.logout { + if sockCred == cred { + copy(socket.logout[i:], socket.logout[i+1:]) + socket.logout = socket.logout[:len(socket.logout)-1] + return true + } + } + return false +} diff --git a/auth_test.go b/auth_test.go index b7298786f..689812477 100644 --- a/auth_test.go +++ b/auth_test.go @@ -46,7 +46,7 @@ import ( func (s *S) TestAuthLoginDatabase(c *C) { // Test both with a normal database and with an authenticated shard. for _, addr := range []string{"localhost:40002", "localhost:40203"} { - session, err := mgo.Dial(addr + expFeaturesString) + session, err := mgo.Dial(addr) c.Assert(err, IsNil) defer session.Close() @@ -70,7 +70,7 @@ func (s *S) TestAuthLoginDatabase(c *C) { func (s *S) TestAuthLoginSession(c *C) { // Test both with a normal database and with an authenticated shard. for _, addr := range []string{"localhost:40002", "localhost:40203"} { - session, err := mgo.Dial(addr + expFeaturesString) + session, err := mgo.Dial(addr) c.Assert(err, IsNil) defer session.Close() @@ -123,7 +123,7 @@ func (s *S) TestAuthLoginLogout(c *C) { } func (s *S) TestAuthLoginLogoutAll(c *C) { - session, err := mgo.Dial("localhost:40002" + expFeaturesString) + session, err := mgo.Dial("localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -171,7 +171,7 @@ func (s *S) TestAuthUpsertUser(c *C) { if !s.versionAtLeast(2, 4) { c.Skip("UpsertUser only works on 2.4+") } - session, err := mgo.Dial("localhost:40002" + expFeaturesString) + session, err := mgo.Dial("localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -252,7 +252,7 @@ func (s *S) TestAuthUpsertUserOtherDBRoles(c *C) { if !s.versionAtLeast(2, 4) { c.Skip("UpsertUser only works on 2.4+") } - session, err := mgo.Dial("localhost:40002" + expFeaturesString) + session, err := mgo.Dial("localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -285,7 +285,7 @@ func (s *S) TestAuthUpsertUserUpdates(c *C) { if !s.versionAtLeast(2, 4) { c.Skip("UpsertUser only works on 2.4+") } - session, err := mgo.Dial("localhost:40002" + expFeaturesString) + session, err := mgo.Dial("localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -313,7 +313,7 @@ func (s *S) TestAuthUpsertUserUpdates(c *C) { c.Assert(err, IsNil) // Login with the new user. - usession, err := mgo.Dial("myruser:mynewpass@localhost:40002/mydb" + expFeaturesString) + usession, err := mgo.Dial("myruser:mynewpass@localhost:40002/mydb") c.Assert(err, IsNil) defer usession.Close() @@ -332,7 +332,7 @@ func (s *S) TestAuthUpsertUserUpdates(c *C) { c.Assert(err, IsNil) // Dial again to ensure the password hasn't changed. - usession, err = mgo.Dial("myruser:mynewpass@localhost:40002/mydb" + expFeaturesString) + usession, err = mgo.Dial("myruser:mynewpass@localhost:40002/mydb") c.Assert(err, IsNil) defer usession.Close() @@ -342,7 +342,7 @@ func (s *S) TestAuthUpsertUserUpdates(c *C) { } func (s *S) TestAuthAddUser(c *C) { - session, err := mgo.Dial("localhost:40002" + expFeaturesString) + session, err := mgo.Dial("localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -373,7 +373,7 @@ func (s *S) TestAuthAddUser(c *C) { } func (s *S) TestAuthAddUserReplaces(c *C) { - session, err := mgo.Dial("localhost:40002" + expFeaturesString) + session, err := mgo.Dial("localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -400,7 +400,7 @@ func (s *S) TestAuthAddUserReplaces(c *C) { } func (s *S) TestAuthRemoveUser(c *C) { - session, err := mgo.Dial("localhost:40002" + expFeaturesString) + session, err := mgo.Dial("localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -421,7 +421,7 @@ func (s *S) TestAuthRemoveUser(c *C) { } func (s *S) TestAuthLoginTwiceDoesNothing(c *C) { - session, err := mgo.Dial("localhost:40002" + expFeaturesString) + session, err := mgo.Dial("localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -439,7 +439,7 @@ func (s *S) TestAuthLoginTwiceDoesNothing(c *C) { } func (s *S) TestAuthLoginLogoutLoginDoesNothing(c *C) { - session, err := mgo.Dial("localhost:40002" + expFeaturesString) + session, err := mgo.Dial("localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -449,17 +449,16 @@ func (s *S) TestAuthLoginLogoutLoginDoesNothing(c *C) { oldStats := mgo.GetStats() - admindb.Logout() // 1 op - err = admindb.Login("root", "rapadura") // 3 op + admindb.Logout() + err = admindb.Login("root", "rapadura") c.Assert(err, IsNil) newStats := mgo.GetStats() - // Logout is flush directly - c.Assert(newStats.SentOps, Equals, oldStats.SentOps+4) + c.Assert(newStats.SentOps, Equals, oldStats.SentOps) } func (s *S) TestAuthLoginSwitchUser(c *C) { - session, err := mgo.Dial("localhost:40002" + expFeaturesString) + session, err := mgo.Dial("localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -486,7 +485,7 @@ func (s *S) TestAuthLoginSwitchUser(c *C) { } func (s *S) TestAuthLoginChangePassword(c *C) { - session, err := mgo.Dial("localhost:40002" + expFeaturesString) + session, err := mgo.Dial("localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -515,7 +514,7 @@ func (s *S) TestAuthLoginChangePassword(c *C) { } func (s *S) TestAuthLoginCachingWithSessionRefresh(c *C) { - session, err := mgo.Dial("localhost:40002" + expFeaturesString) + session, err := mgo.Dial("localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -531,7 +530,7 @@ func (s *S) TestAuthLoginCachingWithSessionRefresh(c *C) { } func (s *S) TestAuthLoginCachingWithSessionCopy(c *C) { - session, err := mgo.Dial("localhost:40002" + expFeaturesString) + session, err := mgo.Dial("localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -548,7 +547,7 @@ func (s *S) TestAuthLoginCachingWithSessionCopy(c *C) { } func (s *S) TestAuthLoginCachingWithSessionClone(c *C) { - session, err := mgo.Dial("localhost:40002" + expFeaturesString) + session, err := mgo.Dial("localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -565,7 +564,7 @@ func (s *S) TestAuthLoginCachingWithSessionClone(c *C) { } func (s *S) TestAuthLoginCachingWithNewSession(c *C) { - session, err := mgo.Dial("localhost:40002" + expFeaturesString) + session, err := mgo.Dial("localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -585,7 +584,7 @@ func (s *S) TestAuthLoginCachingAcrossPool(c *C) { // Logins are cached even when the connection goes back // into the pool. - session, err := mgo.Dial("localhost:40002" + expFeaturesString) + session, err := mgo.Dial("localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -618,9 +617,9 @@ func (s *S) TestAuthLoginCachingAcrossPool(c *C) { err = other.DB("mydb").Login("myuser", "mypass") c.Assert(err, IsNil) - // No more caching, logout is flush directly + // Both logins were cached, so no ops. newStats := mgo.GetStats() - c.Assert(newStats.SentOps, Equals, oldStats.SentOps+6) + c.Assert(newStats.SentOps, Equals, oldStats.SentOps) // And they actually worked. err = other.DB("mydb").C("mycoll").Insert(M{"n": 1}) @@ -636,7 +635,7 @@ func (s *S) TestAuthLoginCachingAcrossPoolWithLogout(c *C) { // Now verify that logouts are properly flushed if they // are not revalidated after leaving the pool. - session, err := mgo.Dial("localhost:40002" + expFeaturesString) + session, err := mgo.Dial("localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -665,12 +664,12 @@ func (s *S) TestAuthLoginCachingAcrossPoolWithLogout(c *C) { oldStats := mgo.GetStats() - err = other.DB("mydb").Login("myuser", "mypass") // 3 op + err = other.DB("mydb").Login("myuser", "mypass") c.Assert(err, IsNil) - // No more caching, logout is flush directly + // Login was cached, so no ops. newStats := mgo.GetStats() - c.Assert(newStats.SentOps, Equals, oldStats.SentOps+3) + c.Assert(newStats.SentOps, Equals, oldStats.SentOps) // Can't write, since root has been implicitly logged out // when the collection went into the pool, and not revalidated. @@ -687,7 +686,7 @@ func (s *S) TestAuthLoginCachingAcrossPoolWithLogout(c *C) { func (s *S) TestAuthEventual(c *C) { // Eventual sessions don't keep sockets around, so they are // an interesting test case. - session, err := mgo.Dial("localhost:40002" + expFeaturesString) + session, err := mgo.Dial("localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -723,7 +722,7 @@ func (s *S) TestAuthEventual(c *C) { } func (s *S) TestAuthURL(c *C) { - session, err := mgo.Dial("mongodb://root:rapadura@localhost:40002/" + expFeaturesString) + session, err := mgo.Dial("mongodb://root:rapadura@localhost:40002/") c.Assert(err, IsNil) defer session.Close() @@ -732,7 +731,7 @@ func (s *S) TestAuthURL(c *C) { } func (s *S) TestAuthURLWrongCredentials(c *C) { - session, err := mgo.Dial("mongodb://root:wrong@localhost:40002/" + expFeaturesString) + session, err := mgo.Dial("mongodb://root:wrong@localhost:40002/") if session != nil { session.Close() } @@ -743,7 +742,7 @@ func (s *S) TestAuthURLWrongCredentials(c *C) { func (s *S) TestAuthURLWithNewSession(c *C) { // When authentication is in the URL, the new session will // actually carry it on as well, even if logged out explicitly. - session, err := mgo.Dial("mongodb://root:rapadura@localhost:40002/" + expFeaturesString) + session, err := mgo.Dial("mongodb://root:rapadura@localhost:40002/") c.Assert(err, IsNil) defer session.Close() @@ -760,7 +759,7 @@ func (s *S) TestAuthURLWithNewSession(c *C) { } func (s *S) TestAuthURLWithDatabase(c *C) { - session, err := mgo.Dial("mongodb://root:rapadura@localhost:40002" + expFeaturesString) + session, err := mgo.Dial("mongodb://root:rapadura@localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -772,9 +771,9 @@ func (s *S) TestAuthURLWithDatabase(c *C) { for i := 0; i < 2; i++ { var url string if i == 0 { - url = "mongodb://myruser:mypass@localhost:40002/mydb" + expFeaturesString + url = "mongodb://myruser:mypass@localhost:40002/mydb" } else { - url = "mongodb://myruser:mypass@localhost:40002/admin?authSource=mydb" + "&" + string(expFeaturesString[1:]) + url = "mongodb://myruser:mypass@localhost:40002/admin?authSource=mydb" } usession, err := mgo.Dial(url) c.Assert(err, IsNil) @@ -798,7 +797,7 @@ func (s *S) TestDefaultDatabase(c *C) { } for _, test := range tests { - session, err := mgo.Dial(test.url + expFeaturesString) + session, err := mgo.Dial(test.url) c.Assert(err, IsNil) defer session.Close() @@ -815,7 +814,7 @@ func (s *S) TestAuthDirect(c *C) { // Direct connections must work to the master and slaves. for _, port := range []string{"40031", "40032", "40033"} { url := fmt.Sprintf("mongodb://root:rapadura@localhost:%s/?connect=direct", port) - session, err := mgo.Dial(url + "&" + string(expFeaturesString[1:])) + session, err := mgo.Dial(url) c.Assert(err, IsNil) defer session.Close() @@ -831,7 +830,7 @@ func (s *S) TestAuthDirectWithLogin(c *C) { // Direct connections must work to the master and slaves. for _, port := range []string{"40031", "40032", "40033"} { url := fmt.Sprintf("mongodb://localhost:%s/?connect=direct", port) - session, err := mgo.Dial(url + "&" + string(expFeaturesString[1:])) + session, err := mgo.Dial(url) c.Assert(err, IsNil) defer session.Close() @@ -859,7 +858,7 @@ func (s *S) TestAuthScramSha1Cred(c *C) { } host := "localhost:40002" c.Logf("Connecting to %s...", host) - session, err := mgo.Dial(host + expFeaturesString) + session, err := mgo.Dial(host) c.Assert(err, IsNil) defer session.Close() @@ -885,7 +884,7 @@ func (s *S) TestAuthScramSha1URL(c *C) { } host := "localhost:40002" c.Logf("Connecting to %s...", host) - session, err := mgo.Dial(fmt.Sprintf("root:rapadura@%s?authMechanism=SCRAM-SHA-1&%s", host, string(expFeaturesString[1:]))) + session, err := mgo.Dial(fmt.Sprintf("root:rapadura@%s?authMechanism=SCRAM-SHA-1", host)) c.Assert(err, IsNil) defer session.Close() @@ -897,7 +896,7 @@ func (s *S) TestAuthScramSha1URL(c *C) { } func (s *S) TestAuthX509Cred(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() binfo, err := session.BuildInfo() @@ -922,9 +921,6 @@ func (s *S) TestAuthX509Cred(c *C) { c.Logf("Connecting to %s...", host) session, err = mgo.DialWithInfo(&mgo.DialInfo{ Addrs: []string{host}, - ExperimentalFeatures: map[string]bool{ - "opmsg": true, - }, DialServer: func(addr *mgo.ServerAddr) (net.Conn, error) { return tls.Dial("tcp", addr.String(), tlsConfig) }, @@ -969,7 +965,7 @@ func (s *S) TestAuthX509Cred(c *C) { } func (s *S) TestAuthX509CredRDNConstruction(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() binfo, err := session.BuildInfo() @@ -996,9 +992,6 @@ func (s *S) TestAuthX509CredRDNConstruction(c *C) { c.Logf("Connecting to %s...", host) session, err = mgo.DialWithInfo(&mgo.DialInfo{ Addrs: []string{host}, - ExperimentalFeatures: map[string]bool{ - "opmsg": true, - }, DialServer: func(addr *mgo.ServerAddr) (net.Conn, error) { return tls.Dial("tcp", addr.String(), tlsConfig) }, diff --git a/bulk_test.go b/bulk_test.go index cc552c614..fa91dc44c 100644 --- a/bulk_test.go +++ b/bulk_test.go @@ -32,7 +32,7 @@ import ( ) func (s *S) TestBulkInsert(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -52,7 +52,7 @@ func (s *S) TestBulkInsert(c *C) { } func (s *S) TestBulkInsertError(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -73,7 +73,7 @@ func (s *S) TestBulkInsertError(c *C) { } func (s *S) TestBulkInsertErrorUnordered(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -100,7 +100,7 @@ func (s *S) TestBulkInsertErrorUnorderedSplitBatch(c *C) { // into the proper size and delivers them one by one. This test ensures that // the behavior of unordered (that is, continue on error) remains correct // when errors happen and there are batches left. - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -132,7 +132,7 @@ func (s *S) TestBulkInsertErrorUnorderedSplitBatch(c *C) { } func (s *S) TestBulkErrorString(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -180,7 +180,7 @@ func (s *S) TestBulkErrorCases_2_6(c *C) { if !s.versionAtLeast(2, 6) { c.Skip("2.4- has poor bulk reporting") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -219,7 +219,7 @@ func (s *S) TestBulkErrorCases_2_4(c *C) { if s.versionAtLeast(2, 6) { c.Skip("2.6+ has better reporting") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -255,7 +255,7 @@ func (s *S) TestBulkErrorCases_2_4(c *C) { } func (s *S) TestBulkErrorCasesOrdered(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -289,7 +289,7 @@ func (s *S) TestBulkErrorCasesOrdered(c *C) { } func (s *S) TestBulkUpdate(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -318,7 +318,7 @@ func (s *S) TestBulkUpdate(c *C) { } func (s *S) TestBulkUpdateOver1000(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -340,7 +340,7 @@ func (s *S) TestBulkUpdateOver1000(c *C) { } func (s *S) TestBulkUpdateError(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -367,7 +367,7 @@ func (s *S) TestBulkUpdateError(c *C) { } func (s *S) TestBulkUpdateErrorUnordered(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -395,7 +395,7 @@ func (s *S) TestBulkUpdateErrorUnordered(c *C) { } func (s *S) TestBulkUpdateAll(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -424,7 +424,7 @@ func (s *S) TestBulkUpdateAll(c *C) { } func (s *S) TestBulkMixedUnordered(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -454,7 +454,7 @@ func (s *S) TestBulkMixedUnordered(c *C) { } func (s *S) TestBulkUpsert(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -478,7 +478,7 @@ func (s *S) TestBulkUpsert(c *C) { } func (s *S) TestBulkRemove(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -502,7 +502,7 @@ func (s *S) TestBulkRemove(c *C) { } func (s *S) TestBulkRemoveAll(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() diff --git a/cluster.go b/cluster.go index 262e7eafb..7fc639c24 100644 --- a/cluster.go +++ b/cluster.go @@ -132,17 +132,15 @@ func (cluster *mongoCluster) removeServer(server *mongoServer) { } type isMasterResult struct { - IsMaster bool - Secondary bool - Primary string - Hosts []string - Passives []string - Tags bson.D - Msg string - SetName string `bson:"setName"` - MaxWireVersion int `bson:"maxWireVersion"` - MaxWriteBatchSize int `bson:"maxWriteBatchSize"` - MaxMessageSizeBytes int `bson:"maxMessageSizeBytes"` + IsMaster bool + Secondary bool + Primary string + Hosts []string + Passives []string + Tags bson.D + Msg string + SetName string `bson:"setName"` + MaxWireVersion int `bson:"maxWireVersion"` } func (cluster *mongoCluster) isMaster(socket *mongoSocket, result *isMasterResult) error { @@ -242,13 +240,11 @@ func (cluster *mongoCluster) syncServer(server *mongoServer) (info *mongoServerI } info = &mongoServerInfo{ - Master: result.IsMaster, - Mongos: result.Msg == "isdbgrid", - Tags: result.Tags, - SetName: result.SetName, - MaxWireVersion: result.MaxWireVersion, - MaxWriteBatchSize: result.MaxWriteBatchSize, - MaxMessageSizeBytes: result.MaxMessageSizeBytes, + Master: result.IsMaster, + Mongos: result.Msg == "isdbgrid", + Tags: result.Tags, + SetName: result.SetName, + MaxWireVersion: result.MaxWireVersion, } hosts = make([]string, 0, 1+len(result.Hosts)+len(result.Passives)) diff --git a/cluster_test.go b/cluster_test.go index 14d0db273..8945e0962 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -42,8 +42,7 @@ import ( ) func (s *S) TestNewSession(c *C) { - - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -97,8 +96,7 @@ func (s *S) TestNewSession(c *C) { } func (s *S) TestCloneSession(c *C) { - - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -163,7 +161,7 @@ func (s *S) TestCloneSession(c *C) { } func (s *S) TestModeStrong(c *C) { - session, err := mgo.Dial("localhost:40012" + expFeaturesString) + session, err := mgo.Dial("localhost:40012") c.Assert(err, IsNil) defer session.Close() @@ -202,7 +200,7 @@ func (s *S) TestModeStrong(c *C) { func (s *S) TestModeMonotonic(c *C) { // Must necessarily connect to a slave, otherwise the // master connection will be available first. - session, err := mgo.Dial("localhost:40012" + expFeaturesString) + session, err := mgo.Dial("localhost:40012") c.Assert(err, IsNil) defer session.Close() @@ -245,7 +243,7 @@ func (s *S) TestModeMonotonicAfterStrong(c *C) { // Test that a strong session shifting to a monotonic // one preserves the socket untouched. - session, err := mgo.Dial("localhost:40012" + expFeaturesString) + session, err := mgo.Dial("localhost:40012") c.Assert(err, IsNil) defer session.Close() @@ -282,7 +280,7 @@ func (s *S) TestModeStrongAfterMonotonic(c *C) { // Must necessarily connect to a slave, otherwise the // master connection will be available first. - session, err := mgo.Dial("localhost:40012" + expFeaturesString) + session, err := mgo.Dial("localhost:40012") c.Assert(err, IsNil) defer session.Close() @@ -317,7 +315,7 @@ func (s *S) TestModeStrongAfterMonotonic(c *C) { func (s *S) TestModeMonotonicWriteOnIteration(c *C) { // Must necessarily connect to a slave, otherwise the // master connection will be available first. - session, err := mgo.Dial("localhost:40012" + expFeaturesString) + session, err := mgo.Dial("localhost:40012") c.Assert(err, IsNil) defer session.Close() @@ -362,7 +360,7 @@ func (s *S) TestModeMonotonicWriteOnIteration(c *C) { func (s *S) TestModeEventual(c *C) { // Must necessarily connect to a slave, otherwise the // master connection will be available first. - session, err := mgo.Dial("localhost:40012" + expFeaturesString) + session, err := mgo.Dial("localhost:40012") c.Assert(err, IsNil) defer session.Close() @@ -400,7 +398,7 @@ func (s *S) TestModeEventualAfterStrong(c *C) { // Test that a strong session shifting to an eventual // one preserves the socket untouched. - session, err := mgo.Dial("localhost:40012" + expFeaturesString) + session, err := mgo.Dial("localhost:40012") c.Assert(err, IsNil) defer session.Close() @@ -439,7 +437,7 @@ func (s *S) TestModeStrongFallover(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40021" + expFeaturesString) + session, err := mgo.Dial("localhost:40021") c.Assert(err, IsNil) defer session.Close() @@ -480,7 +478,7 @@ func (s *S) TestModePrimaryHiccup(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40021" + expFeaturesString) + session, err := mgo.Dial("localhost:40021") c.Assert(err, IsNil) defer session.Close() @@ -531,7 +529,7 @@ func (s *S) TestModeMonotonicFallover(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40021" + expFeaturesString) + session, err := mgo.Dial("localhost:40021") c.Assert(err, IsNil) defer session.Close() @@ -574,7 +572,7 @@ func (s *S) TestModeMonotonicWithSlaveFallover(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40021" + expFeaturesString) + session, err := mgo.Dial("localhost:40021") c.Assert(err, IsNil) defer session.Close() @@ -604,7 +602,7 @@ func (s *S) TestModeMonotonicWithSlaveFallover(c *C) { c.Fatal("Unknown host: ", ssresult.Host) } - session, err = mgo.Dial(addr + expFeaturesString) + session, err = mgo.Dial(addr) c.Assert(err, IsNil) defer session.Close() @@ -653,7 +651,7 @@ func (s *S) TestModeEventualFallover(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40021" + expFeaturesString) + session, err := mgo.Dial("localhost:40021") c.Assert(err, IsNil) defer session.Close() @@ -690,7 +688,7 @@ func (s *S) TestModeSecondaryJustPrimary(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -705,7 +703,7 @@ func (s *S) TestModeSecondaryPreferredJustPrimary(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -721,7 +719,7 @@ func (s *S) TestModeSecondaryPreferredFallover(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -780,7 +778,7 @@ func (s *S) TestModePrimaryPreferredFallover(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -823,7 +821,7 @@ func (s *S) TestModePrimaryFallover(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -850,7 +848,7 @@ func (s *S) TestModeSecondary(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -876,7 +874,7 @@ func (s *S) TestPreserveSocketCountOnSync(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -933,7 +931,7 @@ func (s *S) TestPreserveSocketCountOnSync(c *C) { // single connection was established. func (s *S) TestTopologySyncWithSingleMaster(c *C) { // Use hostname here rather than IP, to make things trickier. - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -957,7 +955,7 @@ func (s *S) TestTopologySyncWithSingleMaster(c *C) { func (s *S) TestTopologySyncWithSlaveSeed(c *C) { // That's supposed to be a slave. Must run discovery // and find out master to insert successfully. - session, err := mgo.Dial("localhost:40012" + expFeaturesString) + session, err := mgo.Dial("localhost:40012") c.Assert(err, IsNil) defer session.Close() @@ -990,7 +988,7 @@ func (s *S) TestSyncTimeout(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1017,7 +1015,7 @@ func (s *S) TestDialWithTimeout(c *C) { started := time.Now() // 40009 isn't used by the test servers. - session, err := mgo.DialWithTimeout("localhost:40009"+expFeaturesString, timeout) + session, err := mgo.DialWithTimeout("localhost:40009", timeout) if session != nil { session.Close() } @@ -1032,7 +1030,7 @@ func (s *S) TestSocketTimeout(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1063,7 +1061,7 @@ func (s *S) TestSocketTimeoutOnDial(c *C) { started := time.Now() - session, err := mgo.DialWithTimeout("localhost:40001"+expFeaturesString, timeout) + session, err := mgo.DialWithTimeout("localhost:40001", timeout) c.Assert(err, ErrorMatches, "no reachable servers") c.Assert(session, IsNil) @@ -1076,7 +1074,7 @@ func (s *S) TestSocketTimeoutOnInactiveSocket(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1131,9 +1129,6 @@ func (s *S) TestDialWithReplicaSetName(c *C) { Addrs: seedList, Timeout: 5 * time.Second, ReplicaSetName: "rs1", - ExperimentalFeatures: map[string]bool{ - "opmsg": true, - }, } session, err := mgo.DialWithInfo(&info) @@ -1163,7 +1158,7 @@ func (s *S) TestDialWithReplicaSetName(c *C) { } func (s *S) TestDirect(c *C) { - session, err := mgo.Dial("localhost:40012?connect=direct" + "&" + string(expFeaturesString[1:])) + session, err := mgo.Dial("localhost:40012?connect=direct") c.Assert(err, IsNil) defer session.Close() @@ -1207,7 +1202,7 @@ func (s *S) TestDirect(c *C) { } func (s *S) TestDirectToUnknownStateMember(c *C) { - session, err := mgo.Dial("localhost:40041?connect=direct" + "&" + string(expFeaturesString[1:])) + session, err := mgo.Dial("localhost:40041?connect=direct") c.Assert(err, IsNil) defer session.Close() @@ -1234,11 +1229,8 @@ func (s *S) TestDirectToUnknownStateMember(c *C) { func (s *S) TestFailFast(c *C) { info := mgo.DialInfo{ - Addrs: []string{"localhost:99999"}, - Timeout: 5 * time.Second, - ExperimentalFeatures: map[string]bool{ - "opmsg": true, - }, + Addrs: []string{"localhost:99999"}, + Timeout: 5 * time.Second, FailFast: true, } @@ -1252,7 +1244,7 @@ func (s *S) TestFailFast(c *C) { func (s *S) countQueries(c *C, server string) (n int) { defer func() { c.Logf("Queries for %q: %d", server, n) }() - session, err := mgo.Dial(server + "?connect=direct" + "&" + string(expFeaturesString[1:])) + session, err := mgo.Dial(server + "?connect=direct") c.Assert(err, IsNil) defer session.Close() session.SetMode(mgo.Monotonic, true) @@ -1274,7 +1266,7 @@ func (s *S) countQueries(c *C, server string) (n int) { func (s *S) countCommands(c *C, server, commandName string) (n int) { defer func() { c.Logf("Queries for %q: %d", server, n) }() - session, err := mgo.Dial(server + "?connect=direct" + "&" + string(expFeaturesString[1:])) + session, err := mgo.Dial(server + "?connect=direct") c.Assert(err, IsNil) defer session.Close() session.SetMode(mgo.Monotonic, true) @@ -1292,7 +1284,7 @@ func (s *S) TestMonotonicSlaveOkFlagWithMongos(c *C) { if s.versionAtLeast(3, 4) { c.Skip("fail on 3.4+ ? ") } - session, err := mgo.Dial("localhost:40021" + expFeaturesString) + session, err := mgo.Dial("localhost:40021") c.Assert(err, IsNil) defer session.Close() @@ -1311,7 +1303,7 @@ func (s *S) TestMonotonicSlaveOkFlagWithMongos(c *C) { s.Stop(":40201") s.StartAll() - mongos, err := mgo.Dial("localhost:40202" + expFeaturesString) + mongos, err := mgo.Dial("localhost:40202") c.Assert(err, IsNil) defer mongos.Close() @@ -1386,7 +1378,7 @@ func (s *S) TestSecondaryModeWithMongos(c *C) { if s.versionAtLeast(3, 4) { c.Skip("fail on 3.4+ ?") } - session, err := mgo.Dial("localhost:40021" + expFeaturesString) + session, err := mgo.Dial("localhost:40021") c.Assert(err, IsNil) defer session.Close() @@ -1405,7 +1397,7 @@ func (s *S) TestSecondaryModeWithMongos(c *C) { s.Stop(":40201") s.StartAll() - mongos, err := mgo.Dial("localhost:40202" + expFeaturesString) + mongos, err := mgo.Dial("localhost:40202") c.Assert(err, IsNil) defer mongos.Close() @@ -1480,7 +1472,7 @@ func (s *S) TestSecondaryModeWithMongosInsert(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40202" + expFeaturesString) + session, err := mgo.Dial("localhost:40202") c.Assert(err, IsNil) defer session.Close() @@ -1501,7 +1493,7 @@ func (s *S) TestRemovalOfClusterMember(c *C) { c.Skip("-fast") } - master, err := mgo.Dial("localhost:40021" + expFeaturesString) + master, err := mgo.Dial("localhost:40021") c.Assert(err, IsNil) defer master.Close() @@ -1582,11 +1574,11 @@ func (s *S) TestPoolLimitSimple(c *C) { var session *mgo.Session var err error if test == 0 { - session, err = mgo.Dial("localhost:40001" + expFeaturesString) + session, err = mgo.Dial("localhost:40001") c.Assert(err, IsNil) session.SetPoolLimit(1) } else { - session, err = mgo.Dial("localhost:40001?maxPoolSize=1" + "&" + string(expFeaturesString[1:])) + session, err = mgo.Dial("localhost:40001?maxPoolSize=1") c.Assert(err, IsNil) } defer session.Close() @@ -1619,7 +1611,7 @@ func (s *S) TestPoolLimitMany(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -1658,7 +1650,7 @@ func (s *S) TestPoolLimitMany(c *C) { } func (s *S) TestSetModeEventualIterBug(c *C) { - session1, err := mgo.Dial("localhost:40011" + expFeaturesString) + session1, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session1.Close() @@ -1682,7 +1674,7 @@ func (s *S) TestSetModeEventualIterBug(c *C) { } } - session2, err := mgo.Dial("localhost:40011" + expFeaturesString) + session2, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session2.Close() @@ -1775,7 +1767,7 @@ func (s *S) TestPrimaryShutdownOnAuthShard(c *C) { } // Dial the shard. - session, err := mgo.Dial("localhost:40203" + expFeaturesString) + session, err := mgo.Dial("localhost:40203") c.Assert(err, IsNil) defer session.Close() @@ -1786,7 +1778,7 @@ func (s *S) TestPrimaryShutdownOnAuthShard(c *C) { c.Assert(err, IsNil) // Dial the replica set to figure the master out. - rs, err := mgo.Dial("root:rapadura@localhost:40031" + expFeaturesString) + rs, err := mgo.Dial("root:rapadura@localhost:40031") c.Assert(err, IsNil) defer rs.Close() @@ -1832,7 +1824,7 @@ func (s *S) TestNearestSecondary(c *C) { rs1c := "127.0.0.1:40013" s.Freeze(rs1b) - session, err := mgo.Dial(rs1a + expFeaturesString) + session, err := mgo.Dial(rs1a) c.Assert(err, IsNil) defer session.Close() @@ -1897,7 +1889,7 @@ func (s *S) TestNearestServer(c *C) { rs1b := "127.0.0.1:40012" rs1c := "127.0.0.1:40013" - session, err := mgo.Dial(rs1a + expFeaturesString) + session, err := mgo.Dial(rs1a) c.Assert(err, IsNil) defer session.Close() @@ -1961,7 +1953,7 @@ func (s *S) TestConnectCloseConcurrency(c *C) { for i := 0; i < n; i++ { go func() { defer wg.Done() - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") if err != nil { c.Fatal(err) } @@ -1977,7 +1969,7 @@ func (s *S) TestSelectServers(c *C) { c.Skip("read preferences introduced in 2.2") } - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -2006,7 +1998,7 @@ func (s *S) TestSelectServersWithMongos(c *C) { c.Skip("fail on 3.4+") } - session, err := mgo.Dial("localhost:40021" + expFeaturesString) + session, err := mgo.Dial("localhost:40021") c.Assert(err, IsNil) defer session.Close() @@ -2037,7 +2029,7 @@ func (s *S) TestSelectServersWithMongos(c *C) { q23a := s.countQueries(c, "localhost:40023") // Do a SlaveOk query through MongoS - mongos, err := mgo.Dial("localhost:40202" + expFeaturesString) + mongos, err := mgo.Dial("localhost:40202") c.Assert(err, IsNil) defer mongos.Close() @@ -2095,7 +2087,7 @@ func (s *S) TestDoNotFallbackToMonotonic(c *C) { c.Skip("failing on 3.2.17+") } - session, err := mgo.Dial("localhost:40012" + expFeaturesString) + session, err := mgo.Dial("localhost:40012") c.Assert(err, IsNil) defer session.Close() diff --git a/gridfs_test.go b/gridfs_test.go index 984ded17c..9fdd0a26f 100644 --- a/gridfs_test.go +++ b/gridfs_test.go @@ -184,7 +184,7 @@ func (s *S) TestGridFSFileDetails(c *C) { } func (s *S) TestGridFSSetUploadDate(c *C) { - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -212,7 +212,7 @@ func (s *S) TestGridFSSetUploadDate(c *C) { } func (s *S) TestGridFSCreateWithChunking(c *C) { - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -293,7 +293,7 @@ func (s *S) TestGridFSCreateWithChunking(c *C) { } func (s *S) TestGridFSAbort(c *C) { - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -330,7 +330,7 @@ func (s *S) TestGridFSAbort(c *C) { } func (s *S) TestGridFSCloseConflict(c *C) { - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -358,7 +358,7 @@ func (s *S) TestGridFSCloseConflict(c *C) { } func (s *S) TestGridFSOpenNotFound(c *C) { - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -375,7 +375,7 @@ func (s *S) TestGridFSOpenNotFound(c *C) { } func (s *S) TestGridFSReadAll(c *C) { - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -412,7 +412,7 @@ func (s *S) TestGridFSReadAll(c *C) { } func (s *S) TestGridFSReadChunking(c *C) { - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -472,7 +472,7 @@ func (s *S) TestGridFSReadChunking(c *C) { } func (s *S) TestGridFSOpen(c *C) { - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -502,7 +502,7 @@ func (s *S) TestGridFSOpen(c *C) { } func (s *S) TestGridFSSeek(c *C) { - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -581,7 +581,7 @@ func (s *S) TestGridFSSeek(c *C) { } func (s *S) TestGridFSRemoveId(c *C) { - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -619,7 +619,7 @@ func (s *S) TestGridFSRemoveId(c *C) { } func (s *S) TestGridFSRemove(c *C) { - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -649,7 +649,7 @@ func (s *S) TestGridFSRemove(c *C) { } func (s *S) TestGridFSOpenNext(c *C) { - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() diff --git a/server.go b/server.go index ce6140941..7ad955255 100644 --- a/server.go +++ b/server.go @@ -36,12 +36,6 @@ import ( "github.com/globalsign/mgo/bson" ) -const ( - // default value for MongoDB 3.6 - defaultWriteBatchSize = 100000 - defaultMaxMessageSizeBytes = 48000000 -) - // --------------------------------------------------------------------------- // Mongo server encapsulation. @@ -73,15 +67,15 @@ func (dial dialer) isSet() bool { } type mongoServerInfo struct { - Master bool - Mongos bool - Tags bson.D - MaxWireVersion int - SetName string - MaxWriteBatchSize int - MaxMessageSizeBytes int + Master bool + Mongos bool + Tags bson.D + MaxWireVersion int + SetName string } +var defaultServerInfo mongoServerInfo + func newServer(addr string, tcpaddr *net.TCPAddr, sync chan bool, dial dialer) *mongoServer { server := &mongoServer{ Addr: addr, @@ -89,11 +83,8 @@ func newServer(addr string, tcpaddr *net.TCPAddr, sync chan bool, dial dialer) * tcpaddr: tcpaddr, sync: sync, dial: dial, - info: &mongoServerInfo{ - MaxWriteBatchSize: defaultWriteBatchSize, - MaxMessageSizeBytes: defaultMaxMessageSizeBytes, - }, - pingValue: time.Hour, // Push it back before an actual ping. + info: &defaultServerInfo, + pingValue: time.Hour, // Push it back before an actual ping. } go server.pinger(true) return server diff --git a/session.go b/session.go index e31323f92..d18ca0869 100644 --- a/session.go +++ b/session.go @@ -103,7 +103,6 @@ type Session struct { queryConfig query bypassValidation bool slaveOk bool - experimental map[string]bool } // Database holds collections of documents @@ -320,7 +319,6 @@ func ParseURL(url string) (*DialInfo, error) { poolLimit := 0 appName := "" readPreferenceMode := Primary - experimental := map[string]bool{} var readPreferenceTagSets []bson.D for _, opt := range uinfo.options { switch opt.key { @@ -342,13 +340,6 @@ func ParseURL(url string) (*DialInfo, error) { return nil, errors.New("appName too long, must be < 128 bytes: " + opt.value) } appName = opt.value - case "experimental": - switch opt.value { - case "opmsg": - experimental[opt.value] = true - default: - return nil, errors.New("unknow experimental feature: " + opt.value) - } case "readPreference": switch opt.value { case "nearest": @@ -408,8 +399,7 @@ func ParseURL(url string) (*DialInfo, error) { Mode: readPreferenceMode, TagSets: readPreferenceTagSets, }, - ReplicaSetName: setName, - ExperimentalFeatures: experimental, + ReplicaSetName: setName, } return &info, nil } @@ -489,12 +479,6 @@ type DialInfo struct { // WARNING: This field is obsolete. See DialServer above. Dial func(addr net.Addr) (net.Conn, error) - - // List of experimental feature to enable. Set the value to 'true' - // to enable a feature. - // Currently, experimental features are: - // - opmsg - ExperimentalFeatures map[string]bool } // ReadPreference defines the manner in which servers are chosen. @@ -585,12 +569,7 @@ func DialWithInfo(info *DialInfo) (*Session, error) { } else { session.SetMode(Strong, true) } - if len(info.ExperimentalFeatures) > 0 { - session.experimental = make(map[string]bool, 0) - for k, v := range info.ExperimentalFeatures { - session.experimental[k] = v - } - } + return session, nil } @@ -682,14 +661,6 @@ func copySession(session *Session, keepCreds bool) (s *Session) { } else if session.dialCred != nil { creds = []Credential{*session.dialCred} } - - var experimental map[string]bool - if len(session.experimental) > 0 { - experimental = make(map[string]bool, len(session.experimental)) - for k, v := range session.experimental { - experimental[k] = v - } - } scopy := Session{ defaultdb: session.defaultdb, sourcedb: session.sourcedb, @@ -707,7 +678,6 @@ func copySession(session *Session, keepCreds bool) (s *Session) { queryConfig: session.queryConfig, bypassValidation: session.bypassValidation, slaveOk: session.slaveOk, - experimental: experimental, } s = &scopy debugf("New session %p on cluster %p (copy from %p)", s, cluster, session) @@ -4966,15 +4936,13 @@ func (iter *Iter) replyFunc() replyFunc { } type writeCmdResult struct { - Ok bool `bson:"ok"` - N int `bson:"n"` - NModified int `bson:"nModified"` + Ok bool + N int + NModified int `bson:"nModified"` Upserted []struct { Index int Id interface{} `bson:"_id"` } - Code int `bson:"code"` - Errmsg string `bson:"errmsg"` ConcernError writeConcernError `bson:"writeConcernError"` Errors []writeCmdError `bson:"writeErrors"` } @@ -4998,140 +4966,6 @@ func (r *writeCmdResult) BulkErrorCases() []BulkErrorCase { return ecases } -func (c *Collection) writeOpWithOpMsg(socket *mongoSocket, serverInfo *mongoServerInfo, op interface{}, ordered, bypassValidation bool, safeOp *queryOp) (*LastError, error) { - var cmd bson.D - var documents []interface{} - var docSeqID string - canUseOpMsg := true - switch msgOp := op.(type) { - case *insertOp: - cmd = bson.D{ - {Name: "insert", Value: c.Name}, - } - docSeqID = "documents" - documents = msgOp.documents - case bulkUpdateOp: - cmd = bson.D{ - {Name: "update", Value: c.Name}, - } - docSeqID = "updates" - documents = msgOp - case bulkDeleteOp: - cmd = bson.D{ - {Name: "delete", Value: c.Name}, - } - docSeqID = "deletes" - documents = msgOp - default: - canUseOpMsg = false - } - - if canUseOpMsg { - //msg flags, see https://docs.mongodb.com/master/reference/mongodb-wire-protocol/#flag-bits - flags := uint32(0) - - var writeConcern interface{} - if safeOp == nil { - // unacknowledged writes - flags |= opMsgFlagMoreToCome - writeConcern = bson.D{{Name: "w", Value: 0}} - } else { - writeConcern = safeOp.query.(*getLastError) - } - - cmd = append(cmd, bson.DocElem{ - Name: "$db", Value: c.Database.Name, - }, bson.DocElem{ - Name: "ordered", Value: ordered, - }, bson.DocElem{ - Name: "writeConcern", Value: writeConcern, - }, bson.DocElem{ - Name: "bypassDocumentValidation", Value: bypassValidation, - }) - - body := msgSection{ - payloadType: msgPayload0, - data: cmd, - } - - n := 0 - modified := 0 - var errs []BulkErrorCase - var lerr LastError - - l := len(documents) - batchNb := (l / serverInfo.MaxWriteBatchSize) + 1 - if l != 0 && (l%serverInfo.MaxWriteBatchSize) == 0 { - batchNb-- - } - count := 0 - - for count < batchNb { - start := count * serverInfo.MaxWriteBatchSize - length := l - start - if length > serverInfo.MaxWriteBatchSize { - length = serverInfo.MaxWriteBatchSize - } - - docs := msgSection{ - payloadType: msgPayload1, - data: payloadType1{ - identifier: docSeqID, - docs: documents[start : start+length], - }, - } - count++ - - // CRC-32 checksum is not implemented in Mongodb 3.6 but - // will be in future release. It's optional, so no need - // to set it for the moment - newOp := &msgOp{ - flags: flags, - sections: []msgSection{body, docs}, - checksum: 0, - } - result, err := socket.sendMessage(newOp) - if err != nil { - return &lerr, err - } - // for some reason, command result format has changed and - // code|errmsg are sometimes top level fields in writeCommandResult - // TODO need to investigate further - if result.Code != 0 { - return &lerr, errors.New(result.Errmsg) - } - if result.ConcernError.Code != 0 { - return &lerr, errors.New(result.ConcernError.ErrMsg) - } - - n += result.N - modified += result.NModified - - if len(result.Errors) > 0 { - for _, e := range result.Errors { - errs = append(errs, BulkErrorCase{ - e.Index, - &QueryError{ - Code: e.Code, - Message: e.ErrMsg, - }, - }) - } - } - } - lerr = LastError{ - N: n, - modified: modified, - ecases: errs, - } - if len(lerr.ecases) > 0 { - return &lerr, lerr.ecases[0].Err - } - return &lerr, nil - } - return nil, nil -} - // writeOp runs the given modifying operation, potentially followed up // by a getLastError command in case the session is in safe mode. The // LastError result is made available in lerr, and if lerr.Err is set it @@ -5147,20 +4981,9 @@ func (c *Collection) writeOp(op interface{}, ordered bool) (lerr *LastError, err s.m.RLock() safeOp := s.safeOp bypassValidation := s.bypassValidation - enableOpMsg := s.experimental["opmsg"] s.m.RUnlock() - serverInfo := socket.ServerInfo() - - if enableOpMsg && serverInfo.MaxWireVersion >= 6 { - // we can use OP_MSG introduced in Mongodb 3.6 - oPlerr, oPerr := c.writeOpWithOpMsg(socket, serverInfo, op, ordered, bypassValidation, safeOp) - if oPlerr != nil || oPerr != nil { - return oPlerr, oPerr - } - } - - if serverInfo.MaxWireVersion >= 2 { + if socket.ServerInfo().MaxWireVersion >= 2 { // Servers with a more recent write protocol benefit from write commands. if op, ok := op.(*insertOp); ok && len(op.documents) > 1000 { var lerr LastError @@ -5440,7 +5263,7 @@ func getRFC2253NameString(RDNElements *pkix.RDNSequence) string { var replacer = strings.NewReplacer(",", "\\,", "=", "\\=", "+", "\\+", "<", "\\<", ">", "\\>", ";", "\\;") //The elements in the sequence needs to be reversed when converting them for i := len(*RDNElements) - 1; i >= 0; i-- { - var nameAndValueList = make([]string, len((*RDNElements)[i])) + var nameAndValueList = make([]string,len((*RDNElements)[i])) for j, attribute := range (*RDNElements)[i] { var shortAttributeName = rdnOIDToShortName(attribute.Type) if len(shortAttributeName) <= 0 { diff --git a/session_test.go b/session_test.go index 84aa1f8a7..eb2c812b3 100644 --- a/session_test.go +++ b/session_test.go @@ -44,7 +44,7 @@ import ( ) func (s *S) TestRunString(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -55,7 +55,7 @@ func (s *S) TestRunString(c *C) { } func (s *S) TestRunValue(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -66,7 +66,7 @@ func (s *S) TestRunValue(c *C) { } func (s *S) TestPing(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -86,19 +86,19 @@ func (s *S) TestPing(c *C) { } func (s *S) TestDialIPAddress(c *C) { - session, err := mgo.Dial("127.0.0.1:40001" + expFeaturesString) + session, err := mgo.Dial("127.0.0.1:40001") c.Assert(err, IsNil) defer session.Close() if os.Getenv("NOIPV6") != "1" { - session, err = mgo.Dial("[::1%]:40001" + expFeaturesString) + session, err = mgo.Dial("[::1%]:40001") c.Assert(err, IsNil) defer session.Close() } } func (s *S) TestURLSingle(c *C) { - session, err := mgo.Dial("mongodb://localhost:40001/" + expFeaturesString) + session, err := mgo.Dial("mongodb://localhost:40001/") c.Assert(err, IsNil) defer session.Close() @@ -109,7 +109,7 @@ func (s *S) TestURLSingle(c *C) { } func (s *S) TestURLMany(c *C) { - session, err := mgo.Dial("mongodb://localhost:40011,localhost:40012/" + expFeaturesString) + session, err := mgo.Dial("mongodb://localhost:40011,localhost:40012/") c.Assert(err, IsNil) defer session.Close() @@ -125,7 +125,7 @@ func (s *S) TestURLParsing(c *C) { "localhost:40001?foo=1;bar=2", } for _, url := range urls { - session, err := mgo.Dial(url + "&" + string(expFeaturesString[1:])) + session, err := mgo.Dial(url) if session != nil { session.Close() } @@ -205,7 +205,7 @@ func (s *S) TestURLWithAppName(c *C) { c.Skip("appName depends on MongoDB 3.4+") } appName := "myAppName" - session, err := mgo.Dial("localhost:40001?appName=" + appName + "&" + string(expFeaturesString[1:])) + session, err := mgo.Dial("localhost:40001?appName=" + appName) c.Assert(err, IsNil) defer session.Close() @@ -240,12 +240,12 @@ func (s *S) TestURLWithAppNameTooLong(c *C) { } appName := "myAppNameWayTooLongmyAppNameWayTooLongmyAppNameWayTooLongmyAppNameWayTooLong" appName += appName - _, err := mgo.Dial("localhost:40001?appName=" + appName + "&" + string(expFeaturesString[1:])) + _, err := mgo.Dial("localhost:40001?appName=" + appName) c.Assert(err, ErrorMatches, "appName too long, must be < 128 bytes: "+appName) } func (s *S) TestInsertFindOne(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -269,7 +269,7 @@ func (s *S) TestInsertFindOne(c *C) { } func (s *S) TestInsertFindOneNil(c *C) { - session, err := mgo.Dial("localhost:40002" + expFeaturesString) + session, err := mgo.Dial("localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -279,7 +279,7 @@ func (s *S) TestInsertFindOneNil(c *C) { } func (s *S) TestInsertFindOneMap(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -294,7 +294,7 @@ func (s *S) TestInsertFindOneMap(c *C) { } func (s *S) TestInsertFindAll(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -340,7 +340,7 @@ func (s *S) TestInsertFindAll(c *C) { } func (s *S) TestFindRef(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -386,7 +386,7 @@ func (s *S) TestFindRef(c *C) { } func (s *S) TestDatabaseAndCollectionNames(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -421,7 +421,7 @@ func (s *S) TestDatabaseAndCollectionNames(c *C) { } func (s *S) TestSelect(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -437,7 +437,7 @@ func (s *S) TestSelect(c *C) { } func (s *S) TestInlineMap(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -468,7 +468,7 @@ func (s *S) TestInlineMap(c *C) { } func (s *S) TestUpdate(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -500,7 +500,7 @@ func (s *S) TestUpdate(c *C) { } func (s *S) TestUpdateId(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -528,7 +528,7 @@ func (s *S) TestUpdateId(c *C) { } func (s *S) TestUpdateNil(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -558,7 +558,7 @@ func (s *S) TestUpdateNil(c *C) { } func (s *S) TestUpsert(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -621,7 +621,7 @@ func (s *S) TestUpsert(c *C) { } func (s *S) TestUpsertId(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -658,7 +658,7 @@ func (s *S) TestUpsertId(c *C) { } func (s *S) TestUpdateAll(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -707,7 +707,7 @@ func (s *S) TestUpdateAll(c *C) { } func (s *S) TestRemove(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -736,7 +736,7 @@ func (s *S) TestRemove(c *C) { } func (s *S) TestRemoveId(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -754,7 +754,7 @@ func (s *S) TestRemoveId(c *C) { } func (s *S) TestRemoveUnsafe(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -774,7 +774,7 @@ func (s *S) TestRemoveUnsafe(c *C) { } func (s *S) TestRemoveAll(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -817,7 +817,7 @@ func (s *S) TestRemoveAll(c *C) { } func (s *S) TestDropDatabase(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -859,7 +859,7 @@ func filterDBs(dbs []string) []string { } func (s *S) TestDropCollection(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -883,7 +883,7 @@ func (s *S) TestDropCollection(c *C) { } func (s *S) TestCreateCollectionCapped(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -909,7 +909,7 @@ func (s *S) TestCreateCollectionCapped(c *C) { } func (s *S) TestCreateCollectionNoIndex(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -929,7 +929,7 @@ func (s *S) TestCreateCollectionNoIndex(c *C) { } func (s *S) TestCreateCollectionForceIndex(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -955,7 +955,7 @@ func (s *S) TestCreateCollectionValidator(c *C) { if !s.versionAtLeast(3, 2) { c.Skip("validation depends on MongoDB 3.2+") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1007,7 +1007,7 @@ func (s *S) TestCreateCollectionStorageEngine(c *C) { if !s.versionAtLeast(3, 0) { c.Skip("storageEngine option depends on MongoDB 3.0+") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1025,7 +1025,7 @@ func (s *S) TestCreateCollectionWithCollation(c *C) { if !s.versionAtLeast(3, 4) { c.Skip("depends on mongodb 3.4+") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1069,7 +1069,7 @@ func (s *S) TestIsDupValues(c *C) { } func (s *S) TestIsDupPrimary(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1083,7 +1083,7 @@ func (s *S) TestIsDupPrimary(c *C) { } func (s *S) TestIsDupUnique(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1105,7 +1105,7 @@ func (s *S) TestIsDupUnique(c *C) { } func (s *S) TestIsDupCapped(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1129,7 +1129,7 @@ func (s *S) TestIsDupCapped(c *C) { } func (s *S) TestIsDupFindAndModify(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1148,7 +1148,7 @@ func (s *S) TestIsDupFindAndModify(c *C) { } func (s *S) TestIsDupRetryUpsert(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1168,7 +1168,7 @@ func (s *S) TestIsDupRetryUpsert(c *C) { } func (s *S) TestFindAndModify(c *C) { - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -1236,7 +1236,7 @@ func (s *S) TestFindAndModify(c *C) { } func (s *S) TestFindAndModifyBug997828(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1258,7 +1258,7 @@ func (s *S) TestFindAndModifyBug997828(c *C) { } func (s *S) TestFindAndModifyErrmsgDoc(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1272,7 +1272,7 @@ func (s *S) TestFindAndModifyErrmsgDoc(c *C) { } func (s *S) TestCountCollection(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1294,7 +1294,7 @@ func (s *S) TestView(c *C) { c.Skip("depends on mongodb 3.4+") } // CreateView has to be run against mongos - session, err := mgo.Dial("localhost:40201" + expFeaturesString) + session, err := mgo.Dial("localhost:40201") c.Assert(err, IsNil) defer session.Close() @@ -1382,12 +1382,16 @@ func (s *S) TestView(c *C) { } func (s *S) TestViewWithCollation(c *C) { - // SERVER-31049 is fixed in 3.4.10 - if !s.versionAtLeast(3, 4, 10) { + // This test is currently failing because of a bug in mongodb. A ticket describing + // the issue is available here: https://jira.mongodb.org/browse/SERVER-31049 + // TODO remove this line when SERVER-31049 is fixed + c.Skip("Fails because of a MongoDB bug as of version 3.4.9, cf https://jira.mongodb.org/browse/SERVER-31049") + + if !s.versionAtLeast(3, 4) { c.Skip("depends on mongodb 3.4+") } // CreateView has to be run against mongos - session, err := mgo.Dial("localhost:40201" + expFeaturesString) + session, err := mgo.Dial("localhost:40201") c.Assert(err, IsNil) defer session.Close() @@ -1416,7 +1420,7 @@ func (s *S) TestViewWithCollation(c *C) { } func (s *S) TestCountQuery(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1434,7 +1438,7 @@ func (s *S) TestCountQuery(c *C) { } func (s *S) TestCountQuerySorted(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1452,7 +1456,7 @@ func (s *S) TestCountQuerySorted(c *C) { } func (s *S) TestCountSkipLimit(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1478,7 +1482,7 @@ func (s *S) TestCountMaxTimeMS(c *C) { c.Skip("SetMaxTime only supported in 2.6+") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1501,7 +1505,7 @@ func (s *S) TestCountHint(c *C) { c.Skip("Not implemented until mongo 2.5.5 https://jira.mongodb.org/browse/SERVER-2677") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1517,7 +1521,7 @@ func (s *S) TestCountHint(c *C) { } func (s *S) TestQueryExplain(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1552,7 +1556,7 @@ func (s *S) TestQueryExplain(c *C) { } func (s *S) TestQuerySetMaxScan(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() coll := session.DB("mydb").C("mycoll") @@ -1575,7 +1579,7 @@ func (s *S) TestQuerySetMaxTime(c *C) { c.Skip("SetMaxTime only supported in 2.6+") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() coll := session.DB("mydb").C("mycoll") @@ -1594,7 +1598,7 @@ func (s *S) TestQuerySetMaxTime(c *C) { } func (s *S) TestQueryHint(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1617,7 +1621,7 @@ func (s *S) TestQueryHint(c *C) { } func (s *S) TestQueryComment(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1663,7 +1667,7 @@ func (s *S) TestQueryComment(c *C) { } func (s *S) TestFindOneNotFound(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1677,7 +1681,7 @@ func (s *S) TestFindOneNotFound(c *C) { } func (s *S) TestFindIterNotFound(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1691,7 +1695,7 @@ func (s *S) TestFindIterNotFound(c *C) { } func (s *S) TestFindNil(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1707,7 +1711,7 @@ func (s *S) TestFindNil(c *C) { } func (s *S) TestFindId(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1725,7 +1729,7 @@ func (s *S) TestFindId(c *C) { } func (s *S) TestFindIterAll(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1771,7 +1775,7 @@ func (s *S) TestFindIterAll(c *C) { } func (s *S) TestFindIterTwiceWithSameQuery(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1797,7 +1801,7 @@ func (s *S) TestFindIterTwiceWithSameQuery(c *C) { } func (s *S) TestFindIterWithoutResults(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1814,7 +1818,7 @@ func (s *S) TestFindIterWithoutResults(c *C) { } func (s *S) TestFindIterLimit(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1866,7 +1870,7 @@ func (s *S) TestResumeIter(c *C) { } const numDocuments = 10 - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") session.SetBatch(4) c.Assert(err, IsNil) defer session.Close() @@ -1928,7 +1932,7 @@ func (s *S) TestFindIterCursorTimeout(c *C) { if !*cursorTimeout { c.Skip("-cursor-timeout") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -1970,7 +1974,7 @@ func (s *S) TestFindIterCursorNoTimeout(c *C) { if !*cursorTimeout { c.Skip("-cursor-timeout") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -2021,7 +2025,7 @@ func (s *S) TestTooManyItemsLimitBug(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(runtime.NumCPU())) @@ -2057,7 +2061,7 @@ func (s *S) TestBatchSizeZeroGetMore(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(runtime.NumCPU())) @@ -2103,7 +2107,7 @@ func (s *S) TestFindIterLimitWithMore(c *C) { if s.versionAtLeast(3, 4) { c.Skip("fail on 3.4+") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -2166,7 +2170,7 @@ func (s *S) TestFindIterLimitWithMore(c *C) { } func (s *S) TestFindIterLimitWithBatch(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -2225,7 +2229,7 @@ func (s *S) TestFindIterLimitWithBatch(c *C) { } func (s *S) TestFindIterSortWithBatch(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -2292,7 +2296,7 @@ func (s *S) TestFindTailTimeoutWithSleep(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -2386,7 +2390,7 @@ func (s *S) TestFindTailTimeoutWithSleep(c *C) { // Test tailable cursors in a situation where Next never gets to sleep once // to respect the timeout requested on Tail. func (s *S) TestFindTailTimeoutNoSleep(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -2471,7 +2475,7 @@ func (s *S) TestFindTailNoTimeout(c *C) { c.Skip("-fast") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -2556,7 +2560,7 @@ func (s *S) TestFindTailNoTimeout(c *C) { } func (s *S) TestIterNextResetsResult(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -2626,7 +2630,7 @@ func (s *S) TestIterNextResetsResult(c *C) { } func (s *S) TestFindForOnIter(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -2678,7 +2682,7 @@ func (s *S) TestFindForOnIter(c *C) { } func (s *S) TestFindFor(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -2730,7 +2734,7 @@ func (s *S) TestFindFor(c *C) { } func (s *S) TestFindForStopOnError(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -2757,7 +2761,7 @@ func (s *S) TestFindForStopOnError(c *C) { } func (s *S) TestFindForResetsResult(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -2828,7 +2832,7 @@ func (s *S) TestFindForResetsResult(c *C) { func (s *S) TestFindIterSnapshot(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -2872,7 +2876,7 @@ func (s *S) TestFindIterSnapshot(c *C) { } func (s *S) TestSort(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -2907,7 +2911,7 @@ func (s *S) TestSort(c *C) { } func (s *S) TestSortWithBadArgs(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -2923,7 +2927,7 @@ func (s *S) TestSortWithBadArgs(c *C) { } func (s *S) TestSortScoreText(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -2983,7 +2987,7 @@ func (s *S) TestSortScoreText(c *C) { } func (s *S) TestPrefetching(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -3072,8 +3076,7 @@ func (s *S) TestPrefetching(c *C) { } func (s *S) TestSafeSetting(c *C) { - - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -3186,7 +3189,7 @@ func (s *S) TestSafeSetting(c *C) { } func (s *S) TestSafeInsert(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -3201,16 +3204,16 @@ func (s *S) TestSafeInsert(c *C) { // Session should be safe by default, so inserting it again must fail. err = coll.Insert(M{"_id": 1}) c.Assert(err, ErrorMatches, ".*E11000 duplicate.*") - if lerr, ok := err.(*mgo.LastError); ok { - c.Assert(lerr.Code, Equals, 11000) - } else { - c.Assert(err.(*mgo.QueryError).Code, Equals, 11000) - } + c.Assert(err.(*mgo.LastError).Code, Equals, 11000) - // It must have sent one operation + // It must have sent two operations (INSERT_OP + getLastError QUERY_OP) stats := mgo.GetStats() - c.Assert(stats.SentOps, Equals, 1) + if s.versionAtLeast(2, 6) { + c.Assert(stats.SentOps, Equals, 1) + } else { + c.Assert(stats.SentOps, Equals, 2) + } mgo.ResetStats() @@ -3225,8 +3228,7 @@ func (s *S) TestSafeInsert(c *C) { } func (s *S) TestSafeParameters(c *C) { - - session, err := mgo.Dial("localhost:40011" + expFeaturesString) + session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -3243,7 +3245,7 @@ func (s *S) TestSafeParameters(c *C) { } func (s *S) TestQueryErrorOne(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -3263,7 +3265,7 @@ func (s *S) TestQueryErrorOne(c *C) { } func (s *S) TestQueryErrorNext(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -3467,7 +3469,7 @@ func getIndex34(session *mgo.Session, db, collection, name string) M { } func (s *S) TestEnsureIndex(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -3571,7 +3573,7 @@ func (s *S) TestEnsureIndex(c *C) { } func (s *S) TestEnsureIndexWithBadInfo(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -3585,7 +3587,7 @@ func (s *S) TestEnsureIndexWithBadInfo(c *C) { } func (s *S) TestEnsureIndexWithUnsafeSession(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -3610,7 +3612,7 @@ func (s *S) TestEnsureIndexWithUnsafeSession(c *C) { } func (s *S) TestEnsureIndexKey(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -3669,7 +3671,7 @@ func (s *S) TestEnsureIndexKey(c *C) { } func (s *S) TestEnsureIndexDropIndex(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -3725,7 +3727,7 @@ func (s *S) TestEnsureIndexDropIndex(c *C) { } func (s *S) TestEnsureIndexDropIndexName(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -3780,7 +3782,7 @@ func (s *S) TestEnsureIndexDropIndexName(c *C) { } func (s *S) TestEnsureIndexDropAllIndexes(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -3813,7 +3815,7 @@ func (s *S) TestEnsureIndexDropAllIndexes(c *C) { } func (s *S) TestEnsureIndexCaching(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -3854,7 +3856,7 @@ func (s *S) TestEnsureIndexCaching(c *C) { } func (s *S) TestEnsureIndexGetIndexes(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -3891,7 +3893,7 @@ func (s *S) TestEnsureIndexGetIndexes(c *C) { } func (s *S) TestEnsureIndexNameCaching(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -3935,7 +3937,7 @@ func (s *S) TestEnsureIndexNameCaching(c *C) { } func (s *S) TestEnsureIndexEvalGetIndexes(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -3972,7 +3974,7 @@ func (s *S) TestEnsureIndexEvalGetIndexes(c *C) { var testTTL = flag.Bool("test-ttl", false, "test TTL collections (may take 1 minute)") func (s *S) TestEnsureIndexExpireAfter(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4020,7 +4022,7 @@ func (s *S) TestEnsureIndexExpireAfter(c *C) { } func (s *S) TestDistinct(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4038,7 +4040,7 @@ func (s *S) TestDistinct(c *C) { } func (s *S) TestMapReduce(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4073,7 +4075,7 @@ func (s *S) TestMapReduce(c *C) { } func (s *S) TestMapReduceFinalize(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4104,7 +4106,7 @@ func (s *S) TestMapReduceFinalize(c *C) { } func (s *S) TestMapReduceToCollection(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4144,7 +4146,7 @@ func (s *S) TestMapReduceToCollection(c *C) { } func (s *S) TestMapReduceToOtherDb(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4184,7 +4186,7 @@ func (s *S) TestMapReduceToOtherDb(c *C) { } func (s *S) TestMapReduceOutOfOrder(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4207,7 +4209,7 @@ func (s *S) TestMapReduceOutOfOrder(c *C) { } func (s *S) TestMapReduceScope(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4229,7 +4231,7 @@ func (s *S) TestMapReduceScope(c *C) { } func (s *S) TestMapReduceVerbose(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4252,7 +4254,7 @@ func (s *S) TestMapReduceVerbose(c *C) { } func (s *S) TestMapReduceLimit(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4274,7 +4276,7 @@ func (s *S) TestMapReduceLimit(c *C) { } func (s *S) TestBuildInfo(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4315,7 +4317,7 @@ func (s *S) TestBuildInfo(c *C) { } func (s *S) TestZeroTimeRoundtrip(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4333,7 +4335,7 @@ func (s *S) TestZeroTimeRoundtrip(c *C) { } func (s *S) TestFsyncLock(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4361,7 +4363,7 @@ func (s *S) TestFsyncLock(c *C) { } func (s *S) TestFsync(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4380,7 +4382,7 @@ func (s *S) TestRepairCursor(c *C) { c.Skip("fail on 3.2.17+") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() session.SetBatch(2) @@ -4427,7 +4429,7 @@ func (s *S) TestPipeIter(c *C) { c.Skip("Pipe only works on 2.1+") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4463,7 +4465,7 @@ func (s *S) TestPipeAll(c *C) { c.Skip("Pipe only works on 2.1+") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4488,7 +4490,7 @@ func (s *S) TestPipeOne(c *C) { c.Skip("Pipe only works on 2.1+") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4513,7 +4515,7 @@ func (s *S) TestPipeExplain(c *C) { c.Skip("Pipe only works on 2.1+") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4530,7 +4532,7 @@ func (s *S) TestPipeExplain(c *C) { } func (s *S) TestBatch1Bug(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4553,7 +4555,7 @@ func (s *S) TestBatch1Bug(c *C) { } func (s *S) TestInterfaceIterBug(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4576,7 +4578,7 @@ func (s *S) TestInterfaceIterBug(c *C) { } func (s *S) TestFindIterCloseKillsCursor(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4597,7 +4599,7 @@ func (s *S) TestFindIterCloseKillsCursor(c *C) { } func (s *S) TestFindIterDoneWithBatches(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4626,7 +4628,7 @@ func (s *S) TestFindIterDoneWithBatches(c *C) { } func (s *S) TestFindIterDoneErr(c *C) { - session, err := mgo.Dial("localhost:40002" + expFeaturesString) + session, err := mgo.Dial("localhost:40002") c.Assert(err, IsNil) defer session.Close() @@ -4641,7 +4643,7 @@ func (s *S) TestFindIterDoneErr(c *C) { } func (s *S) TestFindIterDoneNotFound(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4655,7 +4657,7 @@ func (s *S) TestFindIterDoneNotFound(c *C) { } func (s *S) TestLogReplay(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4676,7 +4678,7 @@ func (s *S) TestLogReplay(c *C) { } func (s *S) TestSetCursorTimeout(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4696,7 +4698,7 @@ func (s *S) TestSetCursorTimeout(c *C) { } func (s *S) TestNewIterNoServer(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4718,7 +4720,7 @@ func (s *S) TestNewIterNoServer(c *C) { } func (s *S) TestNewIterNoServerPresetErr(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4743,7 +4745,7 @@ func (s *S) TestBypassValidation(c *C) { if !s.versionAtLeast(3, 2) { c.Skip("validation supported on 3.2+") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4813,7 +4815,7 @@ func (s *S) TestCollationQueries(c *C) { if !s.versionAtLeast(3, 3, 12) { c.Skip("collations being released with 3.4") } - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4855,7 +4857,7 @@ func (s *S) TestCollationQueries(c *C) { // Some benchmarks that require a running database. func (s *S) BenchmarkFindIterRaw(c *C) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) defer session.Close() @@ -4887,7 +4889,7 @@ func (s *S) BenchmarkFindIterRaw(c *C) { } func BenchmarkInsertSingle(b *testing.B) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") if err != nil { b.Fatal(err) } @@ -4907,7 +4909,7 @@ func BenchmarkInsertSingle(b *testing.B) { } func BenchmarkInsertMultiple(b *testing.B) { - session, err := mgo.Dial("localhost:40001" + expFeaturesString) + session, err := mgo.Dial("localhost:40001") if err != nil { b.Fatal(err) } diff --git a/socket.go b/socket.go index f739baf9c..f6158189c 100644 --- a/socket.go +++ b/socket.go @@ -29,7 +29,6 @@ package mgo import ( "errors" "fmt" - "io" "net" "sync" "time" @@ -39,24 +38,22 @@ import ( type replyFunc func(err error, reply *replyOp, docNum int, docData []byte) -type opMsgReplyFunc func(reply *msgOp, err error) - type mongoSocket struct { sync.Mutex - server *mongoServer // nil when cached - conn net.Conn - timeout time.Duration - addr string // For debugging only. - nextRequestId uint32 - replyFuncs map[uint32]replyFunc - opMsgReplyFuncs map[uint32]opMsgReplyFunc - references int - creds []Credential - cachedNonce string - gotNonce sync.Cond - dead error - serverInfo *mongoServerInfo - closeAfterIdle bool + server *mongoServer // nil when cached + conn net.Conn + timeout time.Duration + addr string // For debugging only. + nextRequestId uint32 + replyFuncs map[uint32]replyFunc + references int + creds []Credential + logout []Credential + cachedNonce string + gotNonce sync.Cond + dead error + serverInfo *mongoServerInfo + closeAfterIdle bool } type queryOpFlags uint32 @@ -68,29 +65,6 @@ const ( flagLogReplay flagNoCursorTimeout flagAwaitData - // section type, as defined here: - // https://docs.mongodb.com/master/reference/mongodb-wire-protocol/#sections - msgPayload0 = uint8(0) - msgPayload1 = uint8(1) - // all possible opCodes, as defined here: - // https://docs.mongodb.com/manual/reference/mongodb-wire-protocol/#request-opcodes - opInvalid = 0 - opReply = 1 - dbMsg = 1000 - dbUpdate = 2001 - dbInsert = 2002 - dbQuery = 2004 - dbGetMore = 2005 - dbDelete = 2006 - dbKillCursors = 2007 - dbCommand = 2010 - dbCommandReply = 2011 - dbCompressed = 2012 - dbMessage = 2013 - // opMsg flags - opMsgFlagChecksumPresent = 1 - opMsgFlagMoreToCome = (1 << 1) - opMsgFlagExhaustAllowed = (1 << 16) ) type queryOp struct { @@ -200,29 +174,6 @@ type killCursorsOp struct { cursorIds []int64 } -type msgSection struct { - payloadType uint8 - data interface{} -} - -// op_msg is introduced in mongodb 3.6, see -// https://docs.mongodb.com/master/reference/mongodb-wire-protocol/#op-msg -// for details -type msgOp struct { - flags uint32 - sections []msgSection - checksum uint32 -} - -// PayloadType1 is a container for the OP_MSG payload data of type 1. -// There is no definition of the type 0 payload because that is simply a -// bson document. -type payloadType1 struct { - size int32 - identifier string - docs []interface{} -} - type requestInfo struct { bufferPos int replyFunc replyFunc @@ -230,11 +181,10 @@ type requestInfo struct { func newSocket(server *mongoServer, conn net.Conn, timeout time.Duration) *mongoSocket { socket := &mongoSocket{ - conn: conn, - addr: server.Addr, - server: server, - replyFuncs: make(map[uint32]replyFunc), - opMsgReplyFuncs: make(map[uint32]opMsgReplyFunc), + conn: conn, + addr: server.Addr, + server: server, + replyFuncs: make(map[uint32]replyFunc), } socket.gotNonce.L = &socket.Mutex if err := socket.InitialAcquire(server.Info(), timeout); err != nil { @@ -402,8 +352,6 @@ func (socket *mongoSocket) kill(err error, abend bool) { stats.socketsAlive(-1) replyFuncs := socket.replyFuncs socket.replyFuncs = make(map[uint32]replyFunc) - opMsgReplyFuncs := socket.opMsgReplyFuncs - socket.opMsgReplyFuncs = make(map[uint32]opMsgReplyFunc) server := socket.server socket.server = nil socket.gotNonce.Broadcast() @@ -412,10 +360,6 @@ func (socket *mongoSocket) kill(err error, abend bool) { logf("Socket %p to %s: notifying replyFunc of closed socket: %s", socket, socket.addr, err.Error()) replyFunc(err, nil, -1, nil) } - for _, opMsgReplyFunc := range opMsgReplyFuncs { - logf("Socket %p to %s: notifying replyFunc of closed socket: %s", socket, socket.addr, err.Error()) - opMsgReplyFunc(nil, err) - } if abend { server.AbendSocket(socket) } @@ -459,8 +403,14 @@ var bytesBufferPool = sync.Pool{ func (socket *mongoSocket) Query(ops ...interface{}) (err error) { - buf := getSizedBuffer(0) - defer bytesBufferPool.Put(buf) + if lops := socket.flushLogout(); len(lops) > 0 { + ops = append(lops, ops...) + } + + buf := bytesBufferPool.Get().([]byte) + defer func() { + bytesBufferPool.Put(buf[:0]) + }() // Serialize operations synchronously to avoid interrupting // other goroutines while we can't really be sending data. @@ -481,7 +431,7 @@ func (socket *mongoSocket) Query(ops ...interface{}) (err error) { switch op := op.(type) { case *updateOp: - buf = addHeader(buf, dbUpdate) + buf = addHeader(buf, 2001) buf = addInt32(buf, 0) // Reserved buf = addCString(buf, op.Collection) buf = addInt32(buf, int32(op.Flags)) @@ -497,7 +447,7 @@ func (socket *mongoSocket) Query(ops ...interface{}) (err error) { } case *insertOp: - buf = addHeader(buf, dbInsert) + buf = addHeader(buf, 2002) buf = addInt32(buf, int32(op.flags)) buf = addCString(buf, op.collection) for _, doc := range op.documents { @@ -509,7 +459,7 @@ func (socket *mongoSocket) Query(ops ...interface{}) (err error) { } case *queryOp: - buf = addHeader(buf, dbQuery) + buf = addHeader(buf, 2004) buf = addInt32(buf, int32(op.flags)) buf = addCString(buf, op.collection) buf = addInt32(buf, op.skip) @@ -527,7 +477,7 @@ func (socket *mongoSocket) Query(ops ...interface{}) (err error) { replyFunc = op.replyFunc case *getMoreOp: - buf = addHeader(buf, dbGetMore) + buf = addHeader(buf, 2005) buf = addInt32(buf, 0) // Reserved buf = addCString(buf, op.collection) buf = addInt32(buf, op.limit) @@ -535,7 +485,7 @@ func (socket *mongoSocket) Query(ops ...interface{}) (err error) { replyFunc = op.replyFunc case *deleteOp: - buf = addHeader(buf, dbDelete) + buf = addHeader(buf, 2006) buf = addInt32(buf, 0) // Reserved buf = addCString(buf, op.Collection) buf = addInt32(buf, int32(op.Flags)) @@ -546,7 +496,7 @@ func (socket *mongoSocket) Query(ops ...interface{}) (err error) { } case *killCursorsOp: - buf = addHeader(buf, dbKillCursors) + buf = addHeader(buf, 2007) buf = addInt32(buf, 0) // Reserved buf = addInt32(buf, int32(len(op.cursorIds))) for _, cursorId := range op.cursorIds { @@ -611,245 +561,123 @@ func (socket *mongoSocket) Query(ops ...interface{}) (err error) { return err } -// sendMessage send data to the database using the OP_MSG wire protocol -// introduced in MongoDB 3.6 (require maxWireVersion >= 6) -func (socket *mongoSocket) sendMessage(op *msgOp) (writeCmdResult, error) { - var wr writeCmdResult - var err error - - buf := getSizedBuffer(0) - defer bytesBufferPool.Put(buf) - - buf = addHeader(buf, dbMessage) - buf = addInt32(buf, int32(op.flags)) - - for _, section := range op.sections { - buf, err = addSection(buf, section) - if err != nil { - return wr, err - } - } - - if len(buf) > socket.ServerInfo().MaxMessageSizeBytes { - return wr, fmt.Errorf("message length to long, should be < %v, but was %v", socket.ServerInfo().MaxMessageSizeBytes, len(buf)) - } - // set the total message size - setInt32(buf, 0, int32(len(buf))) - - var wait sync.Mutex - var reply msgOp - var responseError error - var wcr writeCmdResult - // if no response expected, ie op.flags&opMsgFlagMoreToCome == 1, - // request should have id 0 - var requestID uint32 - // if moreToCome flag is set, we don't want to know the outcome of the message. - // There is no response to a request where moreToCome has been set. - expectReply := (op.flags & opMsgFlagMoreToCome) == 0 - - socket.Lock() - if socket.dead != nil { - dead := socket.dead - socket.Unlock() - debugf("Socket %p to %s: failing query, already closed: %s", socket, socket.addr, socket.dead.Error()) - return wr, dead - } - if expectReply { - // Reserve id 0 for requests which should have no responses. - again: - requestID = socket.nextRequestId + 1 - socket.nextRequestId++ - if requestID == 0 { - goto again - } - wait.Lock() - socket.opMsgReplyFuncs[requestID] = func(msg *msgOp, err error) { - reply = *msg - responseError = err - wait.Unlock() - } - } - socket.Unlock() - - setInt32(buf, 4, int32(requestID)) - stats.sentOps(1) - - socket.updateDeadline(writeDeadline) - _, err = socket.conn.Write(buf) - - if expectReply { - socket.updateDeadline(readDeadline) - wait.Lock() - - if responseError != nil { - return wcr, responseError - } - // for the moment, OP_MSG responses return a body section only, - // cf https://github.com/mongodb/specifications/blob/master/source/message/OP_MSG.rst : - // - // "Similarly, certain commands will reply to messages using this technique when possible - // to avoid the overhead of BSON Arrays. Drivers will be required to allow all command - // replies to use this technique. Drivers will be required to handle Payload Type 1." - // - // so we only return the first section of the response (ie the body) - wcr = reply.sections[0].data.(writeCmdResult) - } - - return wcr, err -} - -// get a slice of byte of `size` length from the pool -func getSizedBuffer(size int) []byte { - b := bytesBufferPool.Get().([]byte) - if len(b) < size { - for i := len(b); i < size; i++ { - b = append(b, byte(0)) - } - return b +func fill(r net.Conn, b []byte) error { + l := len(b) + n, err := r.Read(b) + for n != l && err == nil { + var ni int + ni, err = r.Read(b[n:]) + n += ni } - return b[0:size] + return err } // Estimated minimum cost per socket: 1 goroutine + memory for the largest // document ever seen. func (socket *mongoSocket) readLoop() { - header := make([]byte, 16) // 16 bytes for header - p := make([]byte, 20) // 20 bytes for fixed fields of OP_REPLY + p := make([]byte, 36) // 16 from header + 20 from OP_REPLY fixed fields s := make([]byte, 4) - var r io.Reader = socket.conn // No locking, conn never changes. + conn := socket.conn // No locking, conn never changes. for { - _, err := io.ReadFull(r, header) + err := fill(conn, p) if err != nil { socket.kill(err, true) return } - totalLen := getInt32(header, 0) - responseTo := getInt32(header, 8) - opCode := getInt32(header, 12) + totalLen := getInt32(p, 0) + responseTo := getInt32(p, 8) + opCode := getInt32(p, 12) // Don't use socket.server.Addr here. socket is not // locked and socket.server may go away. debugf("Socket %p to %s: got reply (%d bytes)", socket, socket.addr, totalLen) - stats.receivedOps(1) - switch opCode { - case opReply: - _, err := io.ReadFull(r, p) - if err != nil { - socket.kill(err, true) - return - } - reply := replyOp{ - flags: uint32(getInt32(p, 0)), - cursorId: getInt64(p, 4), - firstDoc: getInt32(p, 12), - replyDocs: getInt32(p, 16), - } - stats.receivedDocs(int(reply.replyDocs)) + _ = totalLen - socket.Lock() - replyFunc, ok := socket.replyFuncs[uint32(responseTo)] - if ok { - delete(socket.replyFuncs, uint32(responseTo)) - } - socket.Unlock() - - if replyFunc != nil && reply.replyDocs == 0 { - replyFunc(nil, &reply, -1, nil) - } else { - for i := 0; i != int(reply.replyDocs); i++ { - _, err := io.ReadFull(r, s) - if err != nil { - if replyFunc != nil { - replyFunc(err, nil, -1, nil) - } - socket.kill(err, true) - return - } - b := getSizedBuffer(int(getInt32(s, 0))) - defer bytesBufferPool.Put(b) - - copy(b[0:4], s) - - _, err = io.ReadFull(r, b[4:]) - if err != nil { - if replyFunc != nil { - replyFunc(err, nil, -1, nil) - } - socket.kill(err, true) - return - } + if opCode != 1 { + socket.kill(errors.New("opcode != 1, corrupted data?"), true) + return + } - if globalDebug && globalLogger != nil { - m := bson.M{} - if err := bson.Unmarshal(b, m); err == nil { - debugf("Socket %p to %s: received document: %#v", socket, socket.addr, m) - } - } + reply := replyOp{ + flags: uint32(getInt32(p, 16)), + cursorId: getInt64(p, 20), + firstDoc: getInt32(p, 28), + replyDocs: getInt32(p, 32), + } + stats.receivedOps(+1) + stats.receivedDocs(int(reply.replyDocs)) + + socket.Lock() + replyFunc, ok := socket.replyFuncs[uint32(responseTo)] + if ok { + delete(socket.replyFuncs, uint32(responseTo)) + } + socket.Unlock() + + if replyFunc != nil && reply.replyDocs == 0 { + replyFunc(nil, &reply, -1, nil) + } else { + for i := 0; i != int(reply.replyDocs); i++ { + err := fill(conn, s) + if err != nil { if replyFunc != nil { - replyFunc(nil, &reply, i, b) + replyFunc(err, nil, -1, nil) } - // XXX Do bound checking against totalLen. + socket.kill(err, true) + return } - } - socket.Lock() - if len(socket.replyFuncs) == 0 { - // Nothing else to read for now. Disable deadline. - socket.conn.SetReadDeadline(time.Time{}) - } else { - socket.updateDeadline(readDeadline) - } - socket.Unlock() + b := make([]byte, int(getInt32(s, 0))) - case dbMessage: - body := getSizedBuffer(int(totalLen) - 16) - defer bytesBufferPool.Put(body) - _, err := io.ReadFull(r, body) - if err != nil { - socket.kill(err, true) - return - } + // copy(b, s) in an efficient way. + b[0] = s[0] + b[1] = s[1] + b[2] = s[2] + b[3] = s[3] - sections, err := getSections(body[4:]) - if err != nil { - socket.kill(err, true) - return - } - // TODO check CRC-32 checksum if checksum byte is set - reply := &msgOp{ - flags: uint32(getInt32(body, 0)), - sections: sections, - } + err = fill(conn, b[4:]) + if err != nil { + if replyFunc != nil { + replyFunc(err, nil, -1, nil) + } + socket.kill(err, true) + return + } - // TODO update this when msgPayload1 section is implemented in MongoDB - stats.receivedDocs(1) - socket.Lock() - opMsgReplyFunc, ok := socket.opMsgReplyFuncs[uint32(responseTo)] - if ok { - delete(socket.opMsgReplyFuncs, uint32(responseTo)) - } - socket.Unlock() + if globalDebug && globalLogger != nil { + m := bson.M{} + if err := bson.Unmarshal(b, m); err == nil { + debugf("Socket %p to %s: received document: %#v", socket, socket.addr, m) + } + } - if opMsgReplyFunc != nil { - opMsgReplyFunc(reply, err) - } else { - socket.kill(fmt.Errorf("couldn't handle response properly"), true) - return + if replyFunc != nil { + replyFunc(nil, &reply, i, b) + } + + // XXX Do bound checking against totalLen. } + } + + socket.Lock() + if len(socket.replyFuncs) == 0 { + // Nothing else to read for now. Disable deadline. socket.conn.SetReadDeadline(time.Time{}) - default: - socket.kill(errors.New("opcode != 1 && opcode != 2013, corrupted data?"), true) - return + } else { + socket.updateDeadline(readDeadline) } + socket.Unlock() + + // XXX Do bound checking against totalLen. } } var emptyHeader = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} -func addHeader(b []byte, opcode int32) []byte { +func addHeader(b []byte, opcode int) []byte { i := len(b) b = append(b, emptyHeader...) // Enough for current opcodes. @@ -873,35 +701,6 @@ func addCString(b []byte, s string) []byte { return b } -// Marshal a section and add it to the provided buffer -// https://docs.mongodb.com/master/reference/mongodb-wire-protocol/#sections -func addSection(b []byte, s msgSection) ([]byte, error) { - var err error - b = append(b, s.payloadType) - switch s.payloadType { - case msgPayload0: - b, err = addBSON(b, s.data) - if err != nil { - return b, err - } - case msgPayload1: - pos := len(b) - b = addInt32(b, 0) - s1 := s.data.(payloadType1) - b = addCString(b, s1.identifier) - for _, doc := range s1.docs { - b, err = bson.MarshalBuffer(doc, b) - if err != nil { - return b, err - } - } - setInt32(b, pos, int32(len(b)-pos)) - default: - return b, fmt.Errorf("invalid section kind in op_msg: %v", s.payloadType) - } - return b, nil -} - func addBSON(b []byte, doc interface{}) ([]byte, error) { if doc == nil { return append(b, 5, 0, 0, 0, 0), nil @@ -937,36 +736,3 @@ func getInt64(b []byte, pos int) int64 { (int64(b[pos+6]) << 48) | (int64(b[pos+7]) << 56) } - -// UnMarshal an array of bytes into a section -// https://docs.mongodb.com/master/reference/mongodb-wire-protocol/#sections -func getSections(b []byte) ([]msgSection, error) { - var sections []msgSection - pos := 0 - for pos != len(b) { - sectionLength := int(getInt32(b, pos+1)) - // first byte is section type - switch b[pos] { - case msgPayload0: - var result writeCmdResult - err := bson.Unmarshal(b[pos+1:pos+sectionLength+1], &result) - if err != nil { - return nil, err - } - sections = append(sections, msgSection{ - payloadType: b[pos], - data: result, - }) - case msgPayload1: - // not implemented yet - // - // b[0:4] size - // b[4:?] docSeqID - // b[?:len(b)] documentSequence - default: - return nil, fmt.Errorf("invalid section type: %v", b[0]) - } - pos += sectionLength + 1 - } - return sections, nil -} diff --git a/suite_test.go b/suite_test.go index e4a149569..624d5a543 100644 --- a/suite_test.go +++ b/suite_test.go @@ -45,8 +45,6 @@ import ( var fast = flag.Bool("fast", false, "Skip slow tests") -var expFeaturesString = "?experimental=opmsg" - type M bson.M type cLogger C From 138ba2fe09fb0f394baea20e82252f40b190ec97 Mon Sep 17 00:00:00 2001 From: Dom Date: Tue, 9 Jan 2018 10:24:23 +0000 Subject: [PATCH 29/38] readme: credit @bachue (#74) --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index a2f608458..c605e6bb0 100644 --- a/README.md +++ b/README.md @@ -32,10 +32,13 @@ Further PR's (with tests) are welcome, but please maintain backwards compatibili * GetBSON correctly handles structs with both fields and pointers ([details](https://github.com/globalsign/mgo/pull/40)) * Improved bson.Raw unmarshalling performance ([details](https://github.com/globalsign/mgo/pull/49)) * Minimise socket connection timeouts due to excessive locking ([details](https://github.com/globalsign/mgo/pull/52)) +* Natively support X509 client authentication ([details](https://github.com/globalsign/mgo/pull/55)) +* Gracefully recover from a temporarily unreachable server ([details](https://github.com/globalsign/mgo/pull/69)) --- ### Thanks to +* @bachue * @bozaro * @BenLubar * @carter2000 From 9acbd68815499fc7f8fbb5931d643b8fd7804a80 Mon Sep 17 00:00:00 2001 From: Dom Date: Tue, 9 Jan 2018 14:05:56 +0000 Subject: [PATCH 30/38] auth: add an example for x509 authentication (#75) * auth: add an example for x509 authentication --- example_test.go | 88 +++++++++++++++++++++++++++++++++++++++++++++++++ session.go | 13 ++++---- 2 files changed, 94 insertions(+), 7 deletions(-) create mode 100644 example_test.go diff --git a/example_test.go b/example_test.go new file mode 100644 index 000000000..1e7ec82e9 --- /dev/null +++ b/example_test.go @@ -0,0 +1,88 @@ +package mgo + +import ( + "crypto/tls" + "crypto/x509" + "io/ioutil" + "net" +) + +func ExampleCredential_x509Authentication() { + // MongoDB follows RFC2253 for the ordering of the DN - if the order is + // incorrect when creating the user in Mongo, the client will not be able to + // connect. + // + // The best way to generate the DN with the correct ordering is with + // openssl: + // + // openssl x509 -in client.crt -inform PEM -noout -subject -nameopt RFC2253 + // subject= CN=Example App,OU=MongoDB Client Authentication,O=GlobalSign,C=GB + // + // + // And then create the user in MongoDB with the above DN: + // + // db.getSiblingDB("$external").runCommand({ + // createUser: "CN=Example App,OU=MongoDB Client Authentication,O=GlobalSign,C=GB", + // roles: [ + // { role: 'readWrite', db: 'bananas' }, + // { role: 'userAdminAnyDatabase', db: 'admin' } + // ], + // writeConcern: { w: "majority" , wtimeout: 5000 } + // }) + // + // + // References: + // - https://docs.mongodb.com/manual/tutorial/configure-x509-client-authentication/ + // - https://docs.mongodb.com/manual/core/security-x.509/ + // + + // Read in the PEM encoded X509 certificate. + // + // See the client.pem file at the path below. + clientCertPEM, err := ioutil.ReadFile("harness/certs/client.pem") + + // Read in the PEM encoded private key. + clientKeyPEM, err := ioutil.ReadFile("harness/certs/client.key") + + // Parse the private key, and the public key contained within the + // certificate. + clientCert, err := tls.X509KeyPair(clientCertPEM, clientKeyPEM) + + // Parse the actual certificate data + clientCert.Leaf, err = x509.ParseCertificate(clientCert.Certificate[0]) + + // Use the cert to set up a TLS connection to Mongo + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{clientCert}, + + // This is set to true so the example works within the test + // environment. + // + // DO NOT set InsecureSkipVerify to true in a production + // environment - if you use an untrusted CA/have your own, load + // its certificate into the RootCAs value instead. + // + // RootCAs: myCAChain, + InsecureSkipVerify: true, + } + + // Connect to Mongo using TLS + host := "localhost:40003" + session, err := DialWithInfo(&DialInfo{ + Addrs: []string{host}, + DialServer: func(addr *ServerAddr) (net.Conn, error) { + return tls.Dial("tcp", host, tlsConfig) + }, + }) + + // Authenticate using the certificate + cred := &Credential{Certificate: tlsConfig.Certificates[0].Leaf} + if err := session.Login(cred); err != nil { + panic(err) + } + + // Done! Use mgo as normal from here. + // + // You should actually check the error code at each step. + _ = err +} diff --git a/session.go b/session.go index d18ca0869..b62707c84 100644 --- a/session.go +++ b/session.go @@ -829,13 +829,12 @@ type Credential struct { // Defaults to "MONGODB-CR". Mechanism string - // Certificate defines an x509 certificate for authentication at login, - // for reference please see, https://docs.mongodb.com/manual/tutorial/configure-x509-client-authentication/ - // If providing a certificate: - // The Username field is populated from the cert and should not be set - // The Mechanism field should be MONGODB-X509 or not set. - // The Source field should be $external or not set. - // If not specified, the username will have to be set manually. + // Certificate sets the x509 certificate for authentication, see: + // + // https://docs.mongodb.com/manual/tutorial/configure-x509-client-authentication/ + // + // If using certificate authentication the Username, Mechanism and Source + // fields should not be set. Certificate *x509.Certificate } From eeedc17ae6c21fc0264fd1439f8bf4ebd2506ab6 Mon Sep 17 00:00:00 2001 From: Dom Date: Mon, 15 Jan 2018 16:52:22 +0000 Subject: [PATCH 31/38] session: add example concurrent usage (#78) --- example_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/example_test.go b/example_test.go index 1e7ec82e9..bf7982a46 100644 --- a/example_test.go +++ b/example_test.go @@ -5,6 +5,7 @@ import ( "crypto/x509" "io/ioutil" "net" + "sync" ) func ExampleCredential_x509Authentication() { @@ -86,3 +87,50 @@ func ExampleCredential_x509Authentication() { // You should actually check the error code at each step. _ = err } + +func ExampleSession_concurrency() { + // This example shows the best practise for concurrent use of a mgo session. + // + // Internally mgo maintains a connection pool, dialling new connections as + // required. + // + // Some general suggestions: + // - Define a struct holding the original session, database name and + // collection name instead of passing them explicitly. + // - Define an interface abstracting your data access instead of exposing + // mgo to your application code directly. + // - Limit concurrency at the application level, not with SetPoolLimit(). + + // This will be our concurrent worker + var doStuff = func(wg *sync.WaitGroup, session *Session) { + defer wg.Done() + + // Copy the session - if needed this will dial a new connection which + // can later be reused. + // + // Calling close returns the connection to the pool. + conn := session.Copy() + defer conn.Close() + + // Do something(s) with the connection + _, _ = conn.DB("").C("my_data").Count() + } + + /////////////////////////////////////////////// + + // Dial a connection to Mongo - this creates the connection pool + session, err := Dial("localhost:40003/my_database") + if err != nil { + panic(err) + } + + // Concurrently do things, passing the session to the worker + wg := &sync.WaitGroup{} + for i := 0; i < 10; i++ { + wg.Add(1) + go doStuff(wg, session) + } + wg.Wait() + + session.Close() +} \ No newline at end of file From 88cedcd1a3f118ef1babdedaa6ab74988b7fc879 Mon Sep 17 00:00:00 2001 From: John Arbash Meinel Date: Thu, 25 Jan 2018 14:05:04 +0400 Subject: [PATCH 32/38] Brings in a patch on having flusher not suppress errors. (#81) https://github.com/go-mgo/mgo/pull/360 --- txn/flusher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/txn/flusher.go b/txn/flusher.go index 3d1882d7f..5d1c1bdd8 100644 --- a/txn/flusher.go +++ b/txn/flusher.go @@ -759,7 +759,7 @@ func (f *flusher) checkpoint(t *transaction, revnos []int64) error { f.debugf("Ready to apply %s. Saving revnos %v: LOST RACE", t, debugRevnos) return f.reload(t) } - return nil + return err } func (f *flusher) apply(t *transaction, pull map[bson.ObjectId]*transaction) error { From 90ad51b6275f11dd72a752048272a44becaef5cf Mon Sep 17 00:00:00 2001 From: Steve Gray Date: Wed, 31 Jan 2018 20:30:54 +1000 Subject: [PATCH 33/38] Fallback to JSON tags when BSON tag isn't present (#91) * Fallback to JSON tags when BSON tag isn't present Cleanup. * Add test to demonstrate tagging fallback. - Test coverage for tagging test. --- bson/bson.go | 16 +++++++++-- bson/compatability_test.go | 54 ++++++++++++++++++++++++++++++++++++++ bson/compatibility.go | 16 +++++++++++ 3 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 bson/compatability_test.go create mode 100644 bson/compatibility.go diff --git a/bson/bson.go b/bson/bson.go index d960f7a37..31beab191 100644 --- a/bson/bson.go +++ b/bson/bson.go @@ -698,9 +698,21 @@ func getStructInfo(st reflect.Type) (*structInfo, error) { info := fieldInfo{Num: i} tag := field.Tag.Get("bson") - if tag == "" && strings.Index(string(field.Tag), ":") < 0 { - tag = string(field.Tag) + + // Fall-back to JSON struct tag, if feature flag is set. + if tag == "" && useJSONTagFallback { + tag = field.Tag.Get("json") } + + // If there's no bson/json tag available. + if tag == "" { + // If there's no tag, and also no tag: value splits (i.e. no colon) + // then assume the entire tag is the value + if strings.Index(string(field.Tag), ":") < 0 { + tag = string(field.Tag) + } + } + if tag == "-" { continue } diff --git a/bson/compatability_test.go b/bson/compatability_test.go new file mode 100644 index 000000000..743a00e8a --- /dev/null +++ b/bson/compatability_test.go @@ -0,0 +1,54 @@ +package bson_test + +import ( + "github.com/globalsign/mgo/bson" + . "gopkg.in/check.v1" +) + +type mixedTagging struct { + First string + Second string `bson:"second_field"` + Third string `json:"third_field"` + Fourth string `bson:"fourth_field" json:"alternate"` +} + +// TestTaggingFallback checks that tagging fallback can be used/works as expected. +func (s *S) TestTaggingFallback(c *C) { + initial := &mixedTagging{ + First: "One", + Second: "Two", + Third: "Three", + Fourth: "Four", + } + + // Take only testing.T, leave only footprints. + initialState := bson.JSONTagFallbackState() + defer bson.SetJSONTagFallback(initialState) + + // Marshal with the new mode applied. + bson.SetJSONTagFallback(true) + bsonState, errBSON := bson.Marshal(initial) + c.Assert(errBSON, IsNil) + + // Unmarshal into a generic map so that we can pick up the actual field names + // selected. + target := make(map[string]string) + errUnmarshal := bson.Unmarshal(bsonState, target) + c.Assert(errUnmarshal, IsNil) + + // No tag, so standard naming + _, firstExists := target["first"] + c.Assert(firstExists, Equals, true) + + // Just a BSON tag + _, secondExists := target["second_field"] + c.Assert(secondExists, Equals, true) + + // Just a JSON tag + _, thirdExists := target["third_field"] + c.Assert(thirdExists, Equals, true) + + // Should marshal 4th as fourth_field (since we have both tags) + _, fourthExists := target["fourth_field"] + c.Assert(fourthExists, Equals, true) +} diff --git a/bson/compatibility.go b/bson/compatibility.go new file mode 100644 index 000000000..6afecf53c --- /dev/null +++ b/bson/compatibility.go @@ -0,0 +1,16 @@ +package bson + +// Current state of the JSON tag fallback option. +var useJSONTagFallback = false + +// SetJSONTagFallback enables or disables the JSON-tag fallback for structure tagging. When this is enabled, structures +// without BSON tags on a field will fall-back to using the JSON tag (if present). +func SetJSONTagFallback(state bool) { + useJSONTagFallback = state +} + +// JSONTagFallbackState returns the current status of the JSON tag fallback compatability option. See SetJSONTagFallback +// for more information. +func JSONTagFallbackState() bool { + return useJSONTagFallback +} From 4eb6ac9ce04db8c661f5cf68108017addfa264d7 Mon Sep 17 00:00:00 2001 From: peterdeka Date: Tue, 6 Feb 2018 16:02:50 +0100 Subject: [PATCH 34/38] Added maxAwaitTimeMS support to ChangeStream so Next() times out. Added tests. Fixed leaking connections and error handling. --- changestreams.go | 63 ++++++- changestreams_test.go | 425 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 479 insertions(+), 9 deletions(-) create mode 100644 changestreams_test.go diff --git a/changestreams.go b/changestreams.go index 369b269e0..b7455347c 100644 --- a/changestreams.go +++ b/changestreams.go @@ -1,13 +1,22 @@ package mgo import ( + "errors" "fmt" "reflect" "sync" + "time" "github.com/globalsign/mgo/bson" ) +type FullDocument string + +const ( + Default = "default" + UpdateLookup = "updateLookup" +) + type ChangeStream struct { iter *Iter isClosed bool @@ -18,13 +27,14 @@ type ChangeStream struct { readPreference *ReadPreference err error m sync.Mutex + sessionCopied bool } type ChangeStreamOptions struct { // FullDocument controls the amount of data that the server will return when // returning a changes document. - FullDocument string + FullDocument FullDocument // ResumeAfter specifies the logical starting point for the new change stream. ResumeAfter *bson.Raw @@ -40,6 +50,8 @@ type ChangeStreamOptions struct { Collation *Collation } +var errMissingResumeToken = errors.New("resume token missing from result") + // Watch constructs a new ChangeStream capable of receiving continuing data // from the database. func (coll *Collection) Watch(pipeline interface{}, @@ -61,7 +73,9 @@ func (coll *Collection) Watch(pipeline interface{}, } pIter.isChangeStream = true - + if options.MaxAwaitTimeMS > 0 { + pIter.timeout = time.Duration(options.MaxAwaitTimeMS) * time.Millisecond + } return &ChangeStream{ iter: pIter, collection: coll, @@ -74,7 +88,8 @@ func (coll *Collection) Watch(pipeline interface{}, // Next retrieves the next document from the change stream, blocking if necessary. // Next returns true if a document was successfully unmarshalled into result, // and false if an error occured. When Next returns false, the Err method should -// be called to check what error occurred during iteration. +// be called to check what error occurred during iteration. If there were no events +// available (ErrNotFound), the Err method returns nil so the user can retry the invocaton. // // For example: // @@ -117,6 +132,13 @@ func (changeStream *ChangeStream) Next(result interface{}) bool { return true } + // if we get no results we return false with no errors so the user can call Next + // again, resuming is not needed as the iterator is simply timed out as no events happened. + // The user will call Timeout in order to understand if this was the case. + if err == ErrNotFound { + return false + } + // check if the error is resumable if !isResumableError(err) { // error is not resumable, give up and return it to the user. @@ -162,6 +184,10 @@ func (changeStream *ChangeStream) Close() error { if err != nil { changeStream.err = err } + if changeStream.sessionCopied { + changeStream.iter.session.Close() + changeStream.sessionCopied = false + } return err } @@ -174,10 +200,15 @@ func (changeStream *ChangeStream) ResumeToken() *bson.Raw { if changeStream.resumeToken == nil { return nil } - var tokenCopy bson.Raw = *changeStream.resumeToken + var tokenCopy = *changeStream.resumeToken return &tokenCopy } +// Timeout returns true if the last call of Next returned false because of an iterator timeout. +func (changeStream *ChangeStream) Timeout() bool { + return changeStream.iter.Timeout() +} + func constructChangeStreamPipeline(pipeline interface{}, options ChangeStreamOptions) interface{} { pipelinev := reflect.ValueOf(pipeline) @@ -197,6 +228,7 @@ func constructChangeStreamPipeline(pipeline interface{}, if options.ResumeAfter != nil { changeStreamStageOptions["resumeAfter"] = options.ResumeAfter } + changeStreamStage := bson.M{"$changeStream": changeStreamStageOptions} pipeOfInterfaces := make([]interface{}, pipelinev.Len()+1) @@ -229,16 +261,29 @@ func (changeStream *ChangeStream) resume() error { } // change out the old connection to the database with the new connection. + if changeStream.sessionCopied { + changeStream.collection.Database.Session.Close() + } changeStream.collection.Database.Session = newSession + changeStream.sessionCopied = true + opts := changeStream.options + if changeStream.resumeToken != nil { + opts.ResumeAfter = changeStream.resumeToken + } // make a new pipeline containing the resume token. - changeStreamPipeline := constructChangeStreamPipeline(changeStream.pipeline, changeStream.options) + changeStreamPipeline := constructChangeStreamPipeline(changeStream.pipeline, opts) // generate the new iterator with the new connection. newPipe := changeStream.collection.Pipe(changeStreamPipeline) changeStream.iter = newPipe.Iter() + if err := changeStream.iter.Err(); err != nil { + return err + } changeStream.iter.isChangeStream = true - + if changeStream.options.MaxAwaitTimeMS > 0 { + changeStream.iter.timeout = time.Duration(changeStream.options.MaxAwaitTimeMS) * time.Millisecond + } return nil } @@ -255,7 +300,7 @@ func (changeStream *ChangeStream) fetchResumeToken(rawResult *bson.Raw) error { } if changeStreamResult.ResumeToken == nil { - return fmt.Errorf("resume token missing from result") + return errMissingResumeToken } changeStream.resumeToken = changeStreamResult.ResumeToken @@ -267,7 +312,6 @@ func (changeStream *ChangeStream) fetchResultSet(result interface{}) error { // fetch the next set of documents from the cursor. gotNext := changeStream.iter.Next(&rawResult) - err := changeStream.iter.Err() if err != nil { return err @@ -295,7 +339,8 @@ func isResumableError(err error) bool { _, isQueryError := err.(*QueryError) // if it is not a database error OR it is a database error, // but the error is a notMaster error - return !isQueryError || isNotMasterError(err) + //and is not a missingResumeToken error (caused by the user provided pipeline) + return (!isQueryError || isNotMasterError(err)) && (err != errMissingResumeToken) } func runKillCursorsOnSession(session *Session, cursorId int64) error { diff --git a/changestreams_test.go b/changestreams_test.go new file mode 100644 index 000000000..5d2a79b17 --- /dev/null +++ b/changestreams_test.go @@ -0,0 +1,425 @@ +package mgo_test + +import ( + mgo "github.com/globalsign/mgo" + "github.com/globalsign/mgo/bson" + . "gopkg.in/check.v1" +) + +type updateDesc struct { + UpdatedFields map[string]interface{} `bson:"updatedFields"` + RemovedFields []string `bson:"removedFields"` +} + +type evNamespace struct { + DB string `bson:"db"` + Coll string `bson:"coll"` +} + +type changeEvent struct { + ID interface{} `bson:"_id"` + OperationType string `bson:"operationType"` + FullDocument *bson.Raw `bson:"fullDocument,omitempty"` + Ns evNamespace `bson:"ns"` + DocumentKey M `bson:"documentKey"` + UpdateDescription *updateDesc `bson:"updateDescription,omitempty"` +} + +func (s *S) TestStreamsWatch(c *C) { + session, err := mgo.Dial("localhost:40011") + c.Assert(err, IsNil) + defer session.Close() + coll := session.DB("mydb").C("mycoll") + //add a mock document + coll.Insert(M{"a": 0}) + + pipeline := []bson.M{} + changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{}) + c.Assert(err, IsNil) + + err = changeStream.Close() + c.Assert(err, IsNil) +} + +func (s *S) TestStreamsInsert(c *C) { + session, err := mgo.Dial("localhost:40011") + c.Assert(err, IsNil) + defer session.Close() + + coll := session.DB("mydb").C("mycoll") + + //add a mock document in order for the DB to be created + err = coll.Insert(M{"a": 0}) + c.Assert(err, IsNil) + + //create the stream + pipeline := []M{} + changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1000}) + c.Assert(err, IsNil) + + //insert a new document + id := bson.NewObjectId() + err = coll.Insert(M{"_id": id, "a": 1}) + c.Assert(err, IsNil) + //get the _id for later check + type A struct { + ID bson.ObjectId `bson:"_id"` + A int `bson:"a"` + } + + //get the event + ev := changeEvent{} + hasEvent := changeStream.Next(&ev) + c.Assert(hasEvent, Equals, true) + + //check event is correct + oid := ev.DocumentKey["_id"].(bson.ObjectId) + c.Assert(oid, Equals, id) + c.Assert(ev.OperationType, Equals, "insert") + c.Assert(ev.FullDocument, NotNil) + a := A{} + err = ev.FullDocument.Unmarshal(&a) + c.Assert(err, IsNil) + c.Assert(a.A, Equals, 1) + c.Assert(ev.Ns.DB, Equals, "mydb") + c.Assert(ev.Ns.Coll, Equals, "mycoll") + + err = changeStream.Close() + c.Assert(err, IsNil) +} + +func (s *S) TestStreamsNextNoEventTimeout(c *C) { + session, err := mgo.Dial("localhost:40011") + c.Assert(err, IsNil) + defer session.Close() + + coll := session.DB("mydb").C("mycoll") + + //add a mock document in order for the DB to be created + id := bson.NewObjectId() + err = coll.Insert(M{"_id": id, "a": 0}) + c.Assert(err, IsNil) + + //create the stream + pipeline := []M{} + changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1000}) + c.Assert(err, IsNil) + + //check we timeout correctly on no events + //we should get a false result and no error + ev := changeEvent{} + hasEvent := changeStream.Next(&ev) + c.Assert(hasEvent, Equals, false) + c.Assert(changeStream.Err(), IsNil) + c.Assert(changeStream.Timeout(), Equals, true) + + err = changeStream.Close() + c.Assert(err, IsNil) +} + +func (s *S) TestStreamsNextTimeout(c *C) { + session, err := mgo.Dial("localhost:40011") + c.Assert(err, IsNil) + defer session.Close() + + coll := session.DB("mydb").C("mycoll") + + //add a mock document in order for the DB to be created + id := bson.NewObjectId() + err = coll.Insert(M{"_id": id, "a": 0}) + c.Assert(err, IsNil) + + //create the stream + pipeline := []M{} + changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 2000}) + c.Assert(err, IsNil) + + //insert a new document to trigger an event + id = bson.NewObjectId() + err = coll.Insert(M{"_id": id, "a": 1}) + c.Assert(err, IsNil) + + //ensure we get the event + ev := changeEvent{} + hasEvent := changeStream.Next(&ev) + c.Assert(hasEvent, Equals, true) + + //check we timeout correctly on no subsequent events + //we should get a false result and no error + ev = changeEvent{} + hasEvent = changeStream.Next(&ev) + c.Assert(hasEvent, Equals, false) + c.Assert(changeStream.Err(), IsNil) + c.Assert(changeStream.Timeout(), Equals, true) + + //insert a new document to trigger an event + id = bson.NewObjectId() + err = coll.Insert(M{"_id": id, "a": 1}) + c.Assert(err, IsNil) + + //ensure we get the event + ev = changeEvent{} + hasEvent = changeStream.Next(&ev) + c.Assert(hasEvent, Equals, true) + + err = changeStream.Close() + c.Assert(err, IsNil) +} + +func (s *S) TestStreamsDelete(c *C) { + session, err := mgo.Dial("localhost:40011") + c.Assert(err, IsNil) + defer session.Close() + + coll := session.DB("mydb").C("mycoll") + + //add a mock document in order for the DB to be created + id := bson.NewObjectId() + err = coll.Insert(M{"_id": id, "a": 0}) + c.Assert(err, IsNil) + + //create the changeStream + pipeline := []M{} + changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1000}) + c.Assert(err, IsNil) + + //delete the document + err = coll.Remove(M{"_id": id}) + c.Assert(err, IsNil) + + //get the event + ev := changeEvent{} + hasEvent := changeStream.Next(&ev) + c.Assert(hasEvent, Equals, true) + + //check event is correct + oid := ev.DocumentKey["_id"].(bson.ObjectId) + c.Assert(oid, Equals, id) + c.Assert(ev.OperationType, Equals, "delete") + c.Assert(ev.FullDocument, IsNil) + c.Assert(ev.Ns.DB, Equals, "mydb") + c.Assert(ev.Ns.Coll, Equals, "mycoll") + + err = changeStream.Close() + c.Assert(err, IsNil) +} + +func (s *S) TestStreamsUpdate(c *C) { + session, err := mgo.Dial("localhost:40011") + c.Assert(err, IsNil) + defer session.Close() + + coll := session.DB("mydb").C("mycoll") + + //add a mock document in order for the DB to be created + id := bson.NewObjectId() + err = coll.Insert(M{"_id": id, "a": 0, "toremove": 2}) + c.Assert(err, IsNil) + + //create the stream + pipeline := []M{} + changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1000}) + c.Assert(err, IsNil) + + //update document + err = coll.UpdateId(id, M{"$set": M{"a": 1}, "$unset": M{"toremove": ""}}) + c.Assert(err, IsNil) + + //get the event + ev := changeEvent{} + hasEvent := changeStream.Next(&ev) + c.Assert(hasEvent, Equals, true) + + //check event is correct + oid := ev.DocumentKey["_id"].(bson.ObjectId) + c.Assert(oid, Equals, id) + c.Assert(ev.OperationType, Equals, "update") + c.Assert(ev.FullDocument, IsNil) + c.Assert(len(ev.UpdateDescription.UpdatedFields), Equals, 1) + c.Assert(len(ev.UpdateDescription.RemovedFields), Equals, 1) + c.Assert(ev.UpdateDescription.UpdatedFields["a"], Equals, 1) + c.Assert(ev.UpdateDescription.RemovedFields[0], Equals, "toremove") + c.Assert(ev.Ns.DB, Equals, "mydb") + c.Assert(ev.Ns.Coll, Equals, "mycoll") + + err = changeStream.Close() + c.Assert(err, IsNil) +} + +func (s *S) TestStreamsUpdateFullDocument(c *C) { + session, err := mgo.Dial("localhost:40011") + c.Assert(err, IsNil) + defer session.Close() + + coll := session.DB("mydb").C("mycoll") + + //add a mock document in order for the DB to be created + id := bson.NewObjectId() + err = coll.Insert(M{"_id": id, "a": 0, "toremove": "bla"}) + c.Assert(err, IsNil) + + //create the stream + pipeline := []M{} + changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1000, FullDocument: mgo.UpdateLookup}) + c.Assert(err, IsNil) + + //update document + err = coll.UpdateId(id, M{"$set": M{"a": 1}, "$unset": M{"toremove": ""}}) + c.Assert(err, IsNil) + + //get the event + ev := changeEvent{} + hasEvent := changeStream.Next(&ev) + c.Assert(hasEvent, Equals, true) + + type A struct { + A int `bson:"a"` + ToRemove *string `bson:"toremove"` + } + + //check event is correct + oid := ev.DocumentKey["_id"].(bson.ObjectId) + c.Assert(oid, Equals, id) + c.Assert(ev.OperationType, Equals, "update") + c.Assert(len(ev.UpdateDescription.UpdatedFields), Equals, 1) + c.Assert(len(ev.UpdateDescription.RemovedFields), Equals, 1) + c.Assert(ev.UpdateDescription.UpdatedFields["a"], Equals, 1) + c.Assert(ev.UpdateDescription.RemovedFields[0], Equals, "toremove") + + c.Assert(ev.FullDocument, NotNil) + a := A{} + err = ev.FullDocument.Unmarshal(&a) + c.Assert(err, IsNil) + c.Assert(a.A, Equals, 1) + c.Assert(a.ToRemove, IsNil) + c.Assert(ev.Ns.DB, Equals, "mydb") + c.Assert(ev.Ns.Coll, Equals, "mycoll") + + err = changeStream.Close() + c.Assert(err, IsNil) +} + +func (s *S) TestStreamsUpdateWithPipeline(c *C) { + session, err := mgo.Dial("localhost:40011") + c.Assert(err, IsNil) + defer session.Close() + + coll := session.DB("mydb").C("mycoll") + + //add two docs + id1 := bson.NewObjectId() + err = coll.Insert(M{"_id": id1, "a": 1}) + c.Assert(err, IsNil) + id2 := bson.NewObjectId() + err = coll.Insert(M{"_id": id2, "a": 2}) + c.Assert(err, IsNil) + + pipeline1 := []M{M{"$match": M{"documentKey._id": id1}}} + changeStream1, err := coll.Watch(pipeline1, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1000}) + c.Assert(err, IsNil) + pipeline2 := []M{M{"$match": M{"documentKey._id": id2}}} + changeStream2, err := coll.Watch(pipeline2, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1000}) + c.Assert(err, IsNil) + + //update documents + _, err = coll.UpdateAll(M{"_id": M{"$in": []bson.ObjectId{id1, id2}}}, M{"$inc": M{"a": 1}}) + c.Assert(err, IsNil) + + got1 := false + got2 := false + + //check we got the update for id1 (and no other) + for i := 0; i < 2; i++ { + ev := changeEvent{} + hasEvent := changeStream1.Next(&ev) + //we will accept only one event, the one that corresponds to our id1 + c.Assert(got1 && hasEvent, Equals, false) + if hasEvent { + oid := ev.DocumentKey["_id"].(bson.ObjectId) + c.Assert(oid, Equals, id1) + got1 = true + } + } + c.Assert(got1, Equals, true) + + //check we got the update for id2 (and no other) + for i := 0; i < 2; i++ { + ev := changeEvent{} + hasEvent := changeStream2.Next(&ev) + //we will accept only one event, the one that corresponds to our id2 + c.Assert(got2 && hasEvent, Equals, false) + if hasEvent { + oid := ev.DocumentKey["_id"].(bson.ObjectId) + c.Assert(oid, Equals, id2) + got2 = true + } + } + c.Assert(got2, Equals, true) + + err = changeStream1.Close() + c.Assert(err, IsNil) + err = changeStream2.Close() + c.Assert(err, IsNil) +} + +func (s *S) TestStreamsResumeTokenMissingError(c *C) { + session, err := mgo.Dial("localhost:40011") + c.Assert(err, IsNil) + defer session.Close() + + coll := session.DB("mydb").C("mycoll") + + //add a mock document in order for the DB to be created + err = coll.Insert(M{"a": 0}) + c.Assert(err, IsNil) + + //create the stream + pipeline := []M{{"$project": M{"_id": 0}}} + changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1000}) + c.Assert(err, IsNil) + + //insert a new document + id := bson.NewObjectId() + err = coll.Insert(M{"_id": id, "a": 1}) + c.Assert(err, IsNil) + + //check we get the correct error + ev := changeEvent{} + hasEvent := changeStream.Next(&ev) + c.Assert(hasEvent, Equals, false) + c.Assert(changeStream.Err().Error(), Equals, "resume token missing from result") + + err = changeStream.Close() + c.Assert(err, IsNil) +} + +func (s *S) TestStreamsClosedStreamError(c *C) { + session, err := mgo.Dial("localhost:40011") + c.Assert(err, IsNil) + defer session.Close() + + coll := session.DB("mydb").C("mycoll") + + //add a mock document in order for the DB to be created + err = coll.Insert(M{"a": 0}) + c.Assert(err, IsNil) + + //create the stream + pipeline := []M{{"$project": M{"_id": 0}}} + changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1000}) + c.Assert(err, IsNil) + + //insert a new document + id := bson.NewObjectId() + err = coll.Insert(M{"_id": id, "a": 1}) + c.Assert(err, IsNil) + + err = changeStream.Close() + c.Assert(err, IsNil) + + //check we get the correct error + ev := changeEvent{} + hasEvent := changeStream.Next(&ev) + c.Assert(hasEvent, Equals, false) + c.Assert(changeStream.Err().Error(), Equals, "illegal use of a closed ChangeStream") +} From 467c79fbaf14932644c77d546cd20bc2df0e991d Mon Sep 17 00:00:00 2001 From: peterdeka Date: Tue, 6 Feb 2018 16:56:29 +0100 Subject: [PATCH 35/38] Enabled journaling in test harness db config that made the ChangeStreams tests fail. --- harness/daemons/.env | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/harness/daemons/.env b/harness/daemons/.env index 7ba8cf599..bc578e07c 100644 --- a/harness/daemons/.env +++ b/harness/daemons/.env @@ -22,7 +22,6 @@ versionAtLeast() { COMMONDOPTSNOIP=" --nohttpinterface - --nojournal --smallfiles --oplogSize=1 --dbpath ./db @@ -59,14 +58,14 @@ if versionAtLeast 3 2; then COMMONDOPTS="$(echo "$COMMONDOPTS" | sed '/--nohttpinterface/d')" COMMONCOPTS="$(echo "$COMMONCOPTS" | sed '/--nohttpinterface/d')" - # config server need to be started as replica set + # config server need to be started as replica set CFG1OPTS="--replSet conf1" CFG2OPTS="--replSet conf2" CFG3OPTS="--replSet conf3" MONGOS1OPTS="--configdb conf1/127.0.0.1:40101" MONGOS2OPTS="--configdb conf2/127.0.0.1:40102" - MONGOS3OPTS="--configdb conf3/127.0.0.1:40103" + MONGOS3OPTS="--configdb conf3/127.0.0.1:40103" fi fi From c8cbfa8e5edb80e333a9ed65667290dddd1d4fbd Mon Sep 17 00:00:00 2001 From: peterdeka Date: Tue, 6 Feb 2018 17:18:46 +0100 Subject: [PATCH 36/38] Better refactored harness daemons env file --nojournal removal. Added mongo version based tests skips for ChangeStreams. --- changestreams_test.go | 30 ++++++++++++++++++++++++++++++ harness/daemons/.env | 8 ++++++++ 2 files changed, 38 insertions(+) diff --git a/changestreams_test.go b/changestreams_test.go index 5d2a79b17..23e78bbba 100644 --- a/changestreams_test.go +++ b/changestreams_test.go @@ -26,6 +26,9 @@ type changeEvent struct { } func (s *S) TestStreamsWatch(c *C) { + if !s.versionAtLeast(3, 6) { + c.Skip("ChangeStreams only work on 3.6+") + } session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -42,6 +45,9 @@ func (s *S) TestStreamsWatch(c *C) { } func (s *S) TestStreamsInsert(c *C) { + if !s.versionAtLeast(3, 6) { + c.Skip("ChangeStreams only work on 3.6+") + } session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -89,6 +95,9 @@ func (s *S) TestStreamsInsert(c *C) { } func (s *S) TestStreamsNextNoEventTimeout(c *C) { + if !s.versionAtLeast(3, 6) { + c.Skip("ChangeStreams only work on 3.6+") + } session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -118,6 +127,9 @@ func (s *S) TestStreamsNextNoEventTimeout(c *C) { } func (s *S) TestStreamsNextTimeout(c *C) { + if !s.versionAtLeast(3, 6) { + c.Skip("ChangeStreams only work on 3.6+") + } session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -167,6 +179,9 @@ func (s *S) TestStreamsNextTimeout(c *C) { } func (s *S) TestStreamsDelete(c *C) { + if !s.versionAtLeast(3, 6) { + c.Skip("ChangeStreams only work on 3.6+") + } session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -205,6 +220,9 @@ func (s *S) TestStreamsDelete(c *C) { } func (s *S) TestStreamsUpdate(c *C) { + if !s.versionAtLeast(3, 6) { + c.Skip("ChangeStreams only work on 3.6+") + } session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -247,6 +265,9 @@ func (s *S) TestStreamsUpdate(c *C) { } func (s *S) TestStreamsUpdateFullDocument(c *C) { + if !s.versionAtLeast(3, 6) { + c.Skip("ChangeStreams only work on 3.6+") + } session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -300,6 +321,9 @@ func (s *S) TestStreamsUpdateFullDocument(c *C) { } func (s *S) TestStreamsUpdateWithPipeline(c *C) { + if !s.versionAtLeast(3, 6) { + c.Skip("ChangeStreams only work on 3.6+") + } session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -363,6 +387,9 @@ func (s *S) TestStreamsUpdateWithPipeline(c *C) { } func (s *S) TestStreamsResumeTokenMissingError(c *C) { + if !s.versionAtLeast(3, 6) { + c.Skip("ChangeStreams only work on 3.6+") + } session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() @@ -394,6 +421,9 @@ func (s *S) TestStreamsResumeTokenMissingError(c *C) { } func (s *S) TestStreamsClosedStreamError(c *C) { + if !s.versionAtLeast(3, 6) { + c.Skip("ChangeStreams only work on 3.6+") + } session, err := mgo.Dial("localhost:40011") c.Assert(err, IsNil) defer session.Close() diff --git a/harness/daemons/.env b/harness/daemons/.env index bc578e07c..a2e7b705d 100644 --- a/harness/daemons/.env +++ b/harness/daemons/.env @@ -22,6 +22,7 @@ versionAtLeast() { COMMONDOPTSNOIP=" --nohttpinterface + --nojournal --smallfiles --oplogSize=1 --dbpath ./db @@ -58,6 +59,13 @@ if versionAtLeast 3 2; then COMMONDOPTS="$(echo "$COMMONDOPTS" | sed '/--nohttpinterface/d')" COMMONCOPTS="$(echo "$COMMONCOPTS" | sed '/--nohttpinterface/d')" + if versionAtLeast 3 6; then + #In version 3.6 --nojournal is deprecated for replica set members using WiredTiger + COMMONDOPTSNOIP="$(echo "$COMMONDOPTSNOIP" | sed '/--nojournal/d')" + COMMONDOPTS="$(echo "$COMMONDOPTS" | sed '/--nojournal/d')" + COMMONCOPTS="$(echo "$COMMONCOPTS" | sed '/--nojournal/d')" + fi + # config server need to be started as replica set CFG1OPTS="--replSet conf1" CFG2OPTS="--replSet conf2" From 5534387743df9ff9441eb3ad65f71e21aceaf41f Mon Sep 17 00:00:00 2001 From: Pietro De Caro Date: Thu, 8 Feb 2018 04:21:02 +0100 Subject: [PATCH 37/38] Changed stream iterator timeout handling from driver to DB Changed stream iterator timeout handling from driver to DB. Added MaxTimeMS option to Pipe in order to be able to set the maximum query run time. Disabled collation parameter in ChangeStreamOptions as currently we don't support it. --- changestreams.go | 26 ++++++++++++-------------- changestreams_test.go | 29 +++++++++++++++++++---------- session.go | 40 +++++++++++++++++++++++++++++++++++----- 3 files changed, 66 insertions(+), 29 deletions(-) diff --git a/changestreams.go b/changestreams.go index b7455347c..371d82851 100644 --- a/changestreams.go +++ b/changestreams.go @@ -5,7 +5,6 @@ import ( "fmt" "reflect" "sync" - "time" "github.com/globalsign/mgo/bson" ) @@ -44,10 +43,10 @@ type ChangeStreamOptions struct { MaxAwaitTimeMS int64 // BatchSize specifies the number of documents to return per batch. - BatchSize int32 + BatchSize int // Collation specifies the way the server should collate returned data. - Collation *Collation + //TODO Collation *Collation } var errMissingResumeToken = errors.New("resume token missing from result") @@ -61,9 +60,15 @@ func (coll *Collection) Watch(pipeline interface{}, pipeline = []bson.M{} } - pipe := constructChangeStreamPipeline(pipeline, options) - - pIter := coll.Pipe(&pipe).Iter() + csPipe := constructChangeStreamPipeline(pipeline, options) + pipe := coll.Pipe(&csPipe) + if options.MaxAwaitTimeMS > 0 { + pipe.MaxTimeMS(options.MaxAwaitTimeMS) + } + if options.BatchSize > 0 { + pipe.Batch(options.BatchSize) + } + pIter := pipe.Iter() // check that there was no issue creating the iterator. // this will fail immediately with an error from the server if running against @@ -73,9 +78,6 @@ func (coll *Collection) Watch(pipeline interface{}, } pIter.isChangeStream = true - if options.MaxAwaitTimeMS > 0 { - pIter.timeout = time.Duration(options.MaxAwaitTimeMS) * time.Millisecond - } return &ChangeStream{ iter: pIter, collection: coll, @@ -248,8 +250,7 @@ func constructChangeStreamPipeline(pipeline interface{}, func (changeStream *ChangeStream) resume() error { // copy the information for the new socket. - // Copy() destroys the sockets currently associated with this session - // so future uses will acquire a new socket against the newly selected DB. + // Thanks to Copy() future uses will acquire a new socket against the newly selected DB. newSession := changeStream.iter.session.Copy() // fetch the cursor from the iterator and use it to run a killCursors @@ -281,9 +282,6 @@ func (changeStream *ChangeStream) resume() error { return err } changeStream.iter.isChangeStream = true - if changeStream.options.MaxAwaitTimeMS > 0 { - changeStream.iter.timeout = time.Duration(changeStream.options.MaxAwaitTimeMS) * time.Millisecond - } return nil } diff --git a/changestreams_test.go b/changestreams_test.go index 23e78bbba..792f5d6ef 100644 --- a/changestreams_test.go +++ b/changestreams_test.go @@ -60,7 +60,7 @@ func (s *S) TestStreamsInsert(c *C) { //create the stream pipeline := []M{} - changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1000}) + changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1500}) c.Assert(err, IsNil) //insert a new document @@ -111,7 +111,7 @@ func (s *S) TestStreamsNextNoEventTimeout(c *C) { //create the stream pipeline := []M{} - changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1000}) + changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1500}) c.Assert(err, IsNil) //check we timeout correctly on no events @@ -122,6 +122,15 @@ func (s *S) TestStreamsNextNoEventTimeout(c *C) { c.Assert(changeStream.Err(), IsNil) c.Assert(changeStream.Timeout(), Equals, true) + //test the same with default timeout (MaxTimeMS=1000) + //create the stream + changeStream, err = coll.Watch(pipeline, mgo.ChangeStreamOptions{}) + c.Assert(err, IsNil) + hasEvent = changeStream.Next(&ev) + c.Assert(hasEvent, Equals, false) + c.Assert(changeStream.Err(), IsNil) + c.Assert(changeStream.Timeout(), Equals, true) + err = changeStream.Close() c.Assert(err, IsNil) } @@ -143,7 +152,7 @@ func (s *S) TestStreamsNextTimeout(c *C) { //create the stream pipeline := []M{} - changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 2000}) + changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1500}) c.Assert(err, IsNil) //insert a new document to trigger an event @@ -195,7 +204,7 @@ func (s *S) TestStreamsDelete(c *C) { //create the changeStream pipeline := []M{} - changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1000}) + changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1500}) c.Assert(err, IsNil) //delete the document @@ -236,7 +245,7 @@ func (s *S) TestStreamsUpdate(c *C) { //create the stream pipeline := []M{} - changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1000}) + changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1500}) c.Assert(err, IsNil) //update document @@ -281,7 +290,7 @@ func (s *S) TestStreamsUpdateFullDocument(c *C) { //create the stream pipeline := []M{} - changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1000, FullDocument: mgo.UpdateLookup}) + changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1500, FullDocument: mgo.UpdateLookup}) c.Assert(err, IsNil) //update document @@ -339,10 +348,10 @@ func (s *S) TestStreamsUpdateWithPipeline(c *C) { c.Assert(err, IsNil) pipeline1 := []M{M{"$match": M{"documentKey._id": id1}}} - changeStream1, err := coll.Watch(pipeline1, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1000}) + changeStream1, err := coll.Watch(pipeline1, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1500}) c.Assert(err, IsNil) pipeline2 := []M{M{"$match": M{"documentKey._id": id2}}} - changeStream2, err := coll.Watch(pipeline2, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1000}) + changeStream2, err := coll.Watch(pipeline2, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1500}) c.Assert(err, IsNil) //update documents @@ -402,7 +411,7 @@ func (s *S) TestStreamsResumeTokenMissingError(c *C) { //create the stream pipeline := []M{{"$project": M{"_id": 0}}} - changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1000}) + changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1500}) c.Assert(err, IsNil) //insert a new document @@ -436,7 +445,7 @@ func (s *S) TestStreamsClosedStreamError(c *C) { //create the stream pipeline := []M{{"$project": M{"_id": 0}}} - changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1000}) + changeStream, err := coll.Watch(pipeline, mgo.ChangeStreamOptions{MaxAwaitTimeMS: 1500}) c.Assert(err, IsNil) //insert a new document diff --git a/session.go b/session.go index f9c9d60c1..ab8515a93 100644 --- a/session.go +++ b/session.go @@ -171,6 +171,7 @@ type Iter struct { timedout bool isFindCmd bool isChangeStream bool + maxTimeMS int64 } var ( @@ -2429,6 +2430,7 @@ type Pipe struct { pipeline interface{} allowDisk bool batchSize int + maxTimeMS int64 } type pipeCmd struct { @@ -2437,6 +2439,7 @@ type pipeCmd struct { Cursor *pipeCmdCursor `bson:",omitempty"` Explain bool `bson:",omitempty"` AllowDisk bool `bson:"allowDiskUse,omitempty"` + MaxTimeMS int64 `bson:"maxTimeMS,omitempty"` } type pipeCmdCursor struct { @@ -2491,6 +2494,9 @@ func (p *Pipe) Iter() *Iter { AllowDisk: p.allowDisk, Cursor: &pipeCmdCursor{p.batchSize}, } + if p.maxTimeMS > 0 { + cmd.MaxTimeMS = p.maxTimeMS + } err := c.Database.Run(cmd, &result) if e, ok := err.(*QueryError); ok && e.Message == `unrecognized field "cursor` { cmd.Cursor = nil @@ -2501,7 +2507,11 @@ func (p *Pipe) Iter() *Iter { if firstBatch == nil { firstBatch = result.Cursor.FirstBatch } - return c.NewIter(p.session, firstBatch, result.Cursor.Id, err) + it := c.NewIter(p.session, firstBatch, result.Cursor.Id, err) + if p.maxTimeMS > 0 { + it.maxTimeMS = p.maxTimeMS + } + return it } // NewIter returns a newly created iterator with the provided parameters. Using @@ -2665,6 +2675,13 @@ func (p *Pipe) Batch(n int) *Pipe { return p } +// MaxTimeMS sets the maximum amount of time to allow the query to run. +// +func (p *Pipe) MaxTimeMS(n int64) *Pipe { + p.maxTimeMS = n + return p +} + // LastError the error status of the preceding write operation on the current connection. // // Relevant documentation: @@ -4021,7 +4038,8 @@ func (iter *Iter) Timeout() bool { // Next returns true if a document was successfully unmarshalled onto result, // and false at the end of the result set or if an error happened. // When Next returns false, the Err method should be called to verify if -// there was an error during iteration. +// there was an error during iteration, and the Timeout method to verify if the +// false return value was caused by a timeout (no available results). // // For example: // @@ -4037,7 +4055,11 @@ func (iter *Iter) Next(result interface{}) bool { iter.m.Lock() iter.timedout = false timeout := time.Time{} - + // for a ChangeStream iterator we have to call getMore before the loop otherwise + // we'll always return false + if iter.isChangeStream { + iter.getMore() + } // check should we expect more data. for iter.err == nil && iter.docData.Len() == 0 && (iter.docsToReceive > 0 || iter.op.cursorId != 0) { // we should expect more data. @@ -4054,6 +4076,12 @@ func (iter *Iter) Next(result interface{}) bool { return false } } + // for a ChangeStream one loop i enought to declare the timeout + if iter.isChangeStream { + iter.timedout = true + iter.m.Unlock() + return false + } // run a getmore to fetch more data. iter.getMore() if iter.err != nil { @@ -4062,7 +4090,6 @@ func (iter *Iter) Next(result interface{}) bool { } iter.gotReply.Wait() } - // We have data from the getMore. // Exhaust available data before reporting any errors. if docData, ok := iter.docData.Pop().([]byte); ok { @@ -4292,6 +4319,9 @@ func (iter *Iter) getMoreCmd() *queryOp { Collection: iter.op.collection[nameDot+1:], BatchSize: iter.op.limit, } + if iter.maxTimeMS > 0 { + getMore.MaxTimeMS = iter.maxTimeMS + } var op queryOp op.collection = iter.op.collection[:nameDot] + ".$cmd" @@ -5276,7 +5306,7 @@ func getRFC2253NameString(RDNElements *pkix.RDNSequence) string { var replacer = strings.NewReplacer(",", "\\,", "=", "\\=", "+", "\\+", "<", "\\<", ">", "\\>", ";", "\\;") //The elements in the sequence needs to be reversed when converting them for i := len(*RDNElements) - 1; i >= 0; i-- { - var nameAndValueList = make([]string,len((*RDNElements)[i])) + var nameAndValueList = make([]string, len((*RDNElements)[i])) for j, attribute := range (*RDNElements)[i] { var shortAttributeName = rdnOIDToShortName(attribute.Type) if len(shortAttributeName) <= 0 { From cebee9afeabd57be9f71d5884aa37bf95ff5e5b9 Mon Sep 17 00:00:00 2001 From: peterdeka Date: Wed, 14 Feb 2018 17:20:33 +0100 Subject: [PATCH 38/38] Renamed Pipe MaxTimeMS to SetMaxTime and changed parameter to a time.Duration --- changestreams.go | 5 +++-- session.go | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/changestreams.go b/changestreams.go index 371d82851..5c2279c66 100644 --- a/changestreams.go +++ b/changestreams.go @@ -5,6 +5,7 @@ import ( "fmt" "reflect" "sync" + "time" "github.com/globalsign/mgo/bson" ) @@ -40,7 +41,7 @@ type ChangeStreamOptions struct { // MaxAwaitTimeMS specifies the maximum amount of time for the server to wait // on new documents to satisfy a change stream query. - MaxAwaitTimeMS int64 + MaxAwaitTimeMS time.Duration // BatchSize specifies the number of documents to return per batch. BatchSize int @@ -63,7 +64,7 @@ func (coll *Collection) Watch(pipeline interface{}, csPipe := constructChangeStreamPipeline(pipeline, options) pipe := coll.Pipe(&csPipe) if options.MaxAwaitTimeMS > 0 { - pipe.MaxTimeMS(options.MaxAwaitTimeMS) + pipe.SetMaxTime(options.MaxAwaitTimeMS) } if options.BatchSize > 0 { pipe.Batch(options.BatchSize) diff --git a/session.go b/session.go index ab8515a93..561f79487 100644 --- a/session.go +++ b/session.go @@ -2675,10 +2675,10 @@ func (p *Pipe) Batch(n int) *Pipe { return p } -// MaxTimeMS sets the maximum amount of time to allow the query to run. +// SetMaxTime sets the maximum amount of time to allow the query to run. // -func (p *Pipe) MaxTimeMS(n int64) *Pipe { - p.maxTimeMS = n +func (p *Pipe) SetMaxTime(d time.Duration) *Pipe { + p.maxTimeMS = int64(d / time.Millisecond) return p }