forked from domodwyer/mgo
-
Notifications
You must be signed in to change notification settings - Fork 230
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Includes: * Reduced memory in bulk operations (#56) * Native x509 authentication (#55) * Better connection recovery (#69) * Example usage (#75 and #78) Thanks to: * @bachue * @csucu * @feliixx --- [Throughput overview](https://user-images.githubusercontent.com/9275968/34954403-3d3253dc-fa18-11e7-8eef-0f2b0f21edc3.png) Select throughput has increased by ~600 requests/second with slightly increased variance: ``` x => r2017.11.06-select-zipfian-throughput.log y => 9acbd68-select-zipfian-throughput.log n min max median average stddev p99 x 3600 49246 71368 66542 66517.26 2327.675 70927.01 y 3600 53304 72005 67151 67145.36 2448.534 71630.00 62000 64000 66000 68000 70000 72000 |----------+-----------+-----------+------------+-----------+-----------+-----| +---------+--------+ 1 -------------------| | |-------------------- +---------+--------+ +---------+---------+ 2 ----------------------------| | |-------------------- +---------+---------+ Legend: 1=data$x, 2=data$y At 95% probablitiy: ===> average is statistically significant (p=0.000000, diff ~628.094444) ===> variance is statistically significant (p=0.002398) ``` * [insert-latency.txt](https://github.com/globalsign/mgo/files/1632474/insert-latency.txt) * [insert-throughput.txt](https://github.com/globalsign/mgo/files/1632475/insert-throughput.txt) * [select-zipfian-latency.txt](https://github.com/globalsign/mgo/files/1632476/select-zipfian-latency.txt) * [select-zipfian-throughput.txt](https://github.com/globalsign/mgo/files/1632477/select-zipfian-throughput.txt) * [update-zipfian-latency.txt](https://github.com/globalsign/mgo/files/1632478/update-zipfian-latency.txt) * [update-zipfian-throughput.txt](https://github.com/globalsign/mgo/files/1632479/update-zipfian-throughput.txt) Note: latencies are approximations calculated from grouped data
- Loading branch information
Showing
14 changed files
with
534 additions
and
163 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,136 @@ | ||
| package mgo | ||
|
|
||
| import ( | ||
| "crypto/tls" | ||
| "crypto/x509" | ||
| "io/ioutil" | ||
| "net" | ||
| "sync" | ||
| ) | ||
|
|
||
| func ExampleCredential_x509Authentication() { | ||
| // MongoDB follows RFC2253 for the ordering of the DN - if the order is | ||
| // incorrect when creating the user in Mongo, the client will not be able to | ||
| // connect. | ||
| // | ||
| // The best way to generate the DN with the correct ordering is with | ||
| // openssl: | ||
| // | ||
| // openssl x509 -in client.crt -inform PEM -noout -subject -nameopt RFC2253 | ||
| // subject= CN=Example App,OU=MongoDB Client Authentication,O=GlobalSign,C=GB | ||
| // | ||
| // | ||
| // And then create the user in MongoDB with the above DN: | ||
| // | ||
| // db.getSiblingDB("$external").runCommand({ | ||
| // createUser: "CN=Example App,OU=MongoDB Client Authentication,O=GlobalSign,C=GB", | ||
| // roles: [ | ||
| // { role: 'readWrite', db: 'bananas' }, | ||
| // { role: 'userAdminAnyDatabase', db: 'admin' } | ||
| // ], | ||
| // writeConcern: { w: "majority" , wtimeout: 5000 } | ||
| // }) | ||
| // | ||
| // | ||
| // References: | ||
| // - https://docs.mongodb.com/manual/tutorial/configure-x509-client-authentication/ | ||
| // - https://docs.mongodb.com/manual/core/security-x.509/ | ||
| // | ||
|
|
||
| // Read in the PEM encoded X509 certificate. | ||
| // | ||
| // See the client.pem file at the path below. | ||
| clientCertPEM, err := ioutil.ReadFile("harness/certs/client.pem") | ||
|
|
||
| // Read in the PEM encoded private key. | ||
| clientKeyPEM, err := ioutil.ReadFile("harness/certs/client.key") | ||
|
|
||
| // Parse the private key, and the public key contained within the | ||
| // certificate. | ||
| clientCert, err := tls.X509KeyPair(clientCertPEM, clientKeyPEM) | ||
|
|
||
| // Parse the actual certificate data | ||
| clientCert.Leaf, err = x509.ParseCertificate(clientCert.Certificate[0]) | ||
|
|
||
| // Use the cert to set up a TLS connection to Mongo | ||
| tlsConfig := &tls.Config{ | ||
| Certificates: []tls.Certificate{clientCert}, | ||
|
|
||
| // This is set to true so the example works within the test | ||
| // environment. | ||
| // | ||
| // DO NOT set InsecureSkipVerify to true in a production | ||
| // environment - if you use an untrusted CA/have your own, load | ||
| // its certificate into the RootCAs value instead. | ||
| // | ||
| // RootCAs: myCAChain, | ||
| InsecureSkipVerify: true, | ||
| } | ||
|
|
||
| // Connect to Mongo using TLS | ||
| host := "localhost:40003" | ||
| session, err := DialWithInfo(&DialInfo{ | ||
| Addrs: []string{host}, | ||
| DialServer: func(addr *ServerAddr) (net.Conn, error) { | ||
| return tls.Dial("tcp", host, tlsConfig) | ||
| }, | ||
| }) | ||
|
|
||
| // Authenticate using the certificate | ||
| cred := &Credential{Certificate: tlsConfig.Certificates[0].Leaf} | ||
| if err := session.Login(cred); err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| // Done! Use mgo as normal from here. | ||
| // | ||
| // You should actually check the error code at each step. | ||
| _ = err | ||
| } | ||
|
|
||
| func ExampleSession_concurrency() { | ||
| // This example shows the best practise for concurrent use of a mgo session. | ||
| // | ||
| // Internally mgo maintains a connection pool, dialling new connections as | ||
| // required. | ||
| // | ||
| // Some general suggestions: | ||
| // - Define a struct holding the original session, database name and | ||
| // collection name instead of passing them explicitly. | ||
| // - Define an interface abstracting your data access instead of exposing | ||
| // mgo to your application code directly. | ||
| // - Limit concurrency at the application level, not with SetPoolLimit(). | ||
|
|
||
| // This will be our concurrent worker | ||
| var doStuff = func(wg *sync.WaitGroup, session *Session) { | ||
| defer wg.Done() | ||
|
|
||
| // Copy the session - if needed this will dial a new connection which | ||
| // can later be reused. | ||
| // | ||
| // Calling close returns the connection to the pool. | ||
| conn := session.Copy() | ||
| defer conn.Close() | ||
|
|
||
| // Do something(s) with the connection | ||
| _, _ = conn.DB("").C("my_data").Count() | ||
| } | ||
|
|
||
| /////////////////////////////////////////////// | ||
|
|
||
| // Dial a connection to Mongo - this creates the connection pool | ||
| session, err := Dial("localhost:40003/my_database") | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| // Concurrently do things, passing the session to the worker | ||
| wg := &sync.WaitGroup{} | ||
| for i := 0; i < 10; i++ { | ||
| wg.Add(1) | ||
| go doStuff(wg, session) | ||
| } | ||
| wg.Wait() | ||
|
|
||
| session.Close() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3,6 +3,5 @@ | |
| . ../.env | ||
|
|
||
| exec mongod $COMMONDOPTS \ | ||
| --shardsvr \ | ||
| --port 40002 \ | ||
| --auth | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3,7 +3,6 @@ | |
| . ../.env | ||
|
|
||
| exec mongod $COMMONDOPTS \ | ||
| --shardsvr \ | ||
| --port 40003 \ | ||
| --auth \ | ||
| --sslMode preferSSL \ | ||
|
|
||
Oops, something went wrong.