This repository has been archived by the owner on Oct 17, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 213
Adapter: Implements RethinkDB as a source of documents #64
Merged
Merged
Changes from 11 commits
Commits
Show all changes
14 commits
Select commit
Hold shift + click to select a range
f3e1798
Initial spike of a RethinkDB sink/destination
alindeman 8ac059c
Adds deletion support for ElasticSearch adapter
alindeman ad9bb49
This method is implemented now
alindeman 0d72ff3
Various cleanups
alindeman 3024b4d
Removes prepareDocument: this is better handled by a transformer script
alindeman e53ceca
Adds sample rethink -> elasticsearch configuration
alindeman eef8994
Reintroduces prepareDocument because the deletion case short circuits…
alindeman 7bc3f1d
Cleans up cursors
alindeman 7f6fc25
Handles errors gracefully
alindeman 06ca331
Removes unvaluable comment
alindeman c2148ae
Makes the configuration struct non-exported
alindeman 4f067fa
Registers the custom configuration type
alindeman 220185c
Returns r instead of nil, per request
alindeman a0f71a9
Tailing RethinkDB requires >= 1.16
alindeman File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,7 @@ | ||
package adaptor | ||
|
||
import ( | ||
"errors" | ||
"fmt" | ||
"net/url" | ||
"strings" | ||
|
@@ -22,6 +23,7 @@ type Rethinkdb struct { | |
table string | ||
|
||
debug bool | ||
tail bool | ||
|
||
// | ||
pipe *pipe.Pipe | ||
|
@@ -31,10 +33,24 @@ type Rethinkdb struct { | |
client *gorethink.Session | ||
} | ||
|
||
// rethinkDbConfig provides custom configuration options for the RethinkDB adapter | ||
type rethinkDbConfig struct { | ||
URI string `json:"uri" doc:"the uri to connect to, in the form rethink://user:[email protected]:28015/database"` | ||
Namespace string `json:"namespace" doc:"rethink namespace to read/write, in the form database.table"` | ||
Debug bool `json:"debug" doc:"if true, verbose debugging information is displayed"` | ||
Tail bool `json:"tail" doc:"if true, the RethinkDB table will be monitored for changes after copying the namespace"` | ||
} | ||
|
||
type rethinkDbChangeNotification struct { | ||
Error string `gorethink:"error"` | ||
OldVal map[string]interface{} `gorethink:"old_val"` | ||
NewVal map[string]interface{} `gorethink:"new_val"` | ||
} | ||
|
||
// NewRethinkdb creates a new Rethinkdb database adaptor | ||
func NewRethinkdb(p *pipe.Pipe, path string, extra Config) (StopStartListener, error) { | ||
var ( | ||
conf dbConfig | ||
conf rethinkDbConfig | ||
err error | ||
) | ||
if err = extra.Construct(&conf); err != nil { | ||
|
@@ -46,10 +62,15 @@ func NewRethinkdb(p *pipe.Pipe, path string, extra Config) (StopStartListener, e | |
return nil, err | ||
} | ||
|
||
if conf.Debug { | ||
fmt.Printf("rethinkDbConfig: %#v\n", conf) | ||
} | ||
|
||
r := &Rethinkdb{ | ||
uri: u, | ||
pipe: p, | ||
path: path, | ||
tail: conf.Tail, | ||
} | ||
|
||
r.database, r.table, err = extra.splitNamespace() | ||
|
@@ -58,22 +79,131 @@ func NewRethinkdb(p *pipe.Pipe, path string, extra Config) (StopStartListener, e | |
} | ||
r.debug = conf.Debug | ||
|
||
r.client, err = gorethink.Connect(gorethink.ConnectOpts{ | ||
Address: r.uri.Host, | ||
MaxIdle: 10, | ||
Timeout: time.Second * 10, | ||
}) | ||
if err != nil { | ||
return nil, err | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Go ahead and return |
||
} | ||
r.client.Use(r.database) | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can you add some version detection here for |
||
return r, nil | ||
} | ||
|
||
// Start the adaptor as a source (not implemented) | ||
// Start the adaptor as a source | ||
func (r *Rethinkdb) Start() error { | ||
return fmt.Errorf("rethinkdb can't function as a source") | ||
} | ||
if r.debug { | ||
fmt.Printf("getting a changes cursor\n") | ||
} | ||
|
||
// Listen start's the adaptor's listener | ||
func (r *Rethinkdb) Listen() (err error) { | ||
r.client, err = r.setupClient() | ||
// Grab a changes cursor before sending all rows. The server will buffer | ||
// changes while we reindex the entire table. | ||
var ccursor *gorethink.Cursor | ||
ccursor, err := gorethink.Table(r.table).Changes().Run(r.client) | ||
if err != nil { | ||
r.pipe.Err <- err | ||
return err | ||
} | ||
defer ccursor.Close() | ||
|
||
if err := r.sendAllDocuments(); err != nil { | ||
r.pipe.Err <- err | ||
return err | ||
} | ||
|
||
if r.tail { | ||
if err := r.sendChanges(ccursor); err != nil { | ||
r.pipe.Err <- err | ||
return err | ||
} | ||
} | ||
|
||
return nil | ||
} | ||
|
||
func (r *Rethinkdb) sendAllDocuments() error { | ||
if r.debug { | ||
fmt.Printf("sending all documents\n") | ||
} | ||
|
||
cursor, err := gorethink.Table(r.table).Run(r.client) | ||
if err != nil { | ||
return err | ||
} | ||
defer cursor.Close() | ||
|
||
var doc map[string]interface{} | ||
for cursor.Next(&doc) { | ||
if stop := r.pipe.Stopped; stop { | ||
return nil | ||
} | ||
|
||
msg := message.NewMsg(message.Insert, r.prepareDocument(doc)) | ||
r.pipe.Send(msg) | ||
} | ||
|
||
if err := cursor.Err(); err != nil { | ||
return err | ||
} | ||
|
||
return nil | ||
} | ||
|
||
func (r *Rethinkdb) sendChanges(ccursor *gorethink.Cursor) error { | ||
if r.debug { | ||
fmt.Printf("sending changes\n") | ||
} | ||
|
||
var change rethinkDbChangeNotification | ||
for ccursor.Next(&change) { | ||
if stop := r.pipe.Stopped; stop { | ||
return nil | ||
} | ||
|
||
if r.debug { | ||
fmt.Printf("change: %#v\n", change) | ||
} | ||
|
||
var msg *message.Msg | ||
if change.Error != "" { | ||
return errors.New(change.Error) | ||
} else if change.OldVal != nil && change.NewVal != nil { | ||
msg = message.NewMsg(message.Update, r.prepareDocument(change.NewVal)) | ||
} else if change.NewVal != nil { | ||
msg = message.NewMsg(message.Insert, r.prepareDocument(change.NewVal)) | ||
} else if change.OldVal != nil { | ||
msg = message.NewMsg(message.Delete, r.prepareDocument(change.OldVal)) | ||
} | ||
|
||
if msg != nil { | ||
fmt.Printf("msg: %#v\n", msg) | ||
r.pipe.Send(msg) | ||
} | ||
} | ||
|
||
if err := ccursor.Err(); err != nil { | ||
return err | ||
} | ||
|
||
return nil | ||
} | ||
|
||
// prepareDocument moves the `id` field to the `_id` field, which is more | ||
// commonly used by downstream sinks. A transformer could be used to do the | ||
// same thing, but because transformers are not run for Delete messages, we | ||
// must do it here. | ||
func (r *Rethinkdb) prepareDocument(doc map[string]interface{}) map[string]interface{} { | ||
doc["_id"] = doc["id"] | ||
delete(doc, "id") | ||
|
||
return doc | ||
} | ||
|
||
// Listen start's the adaptor's listener | ||
func (r *Rethinkdb) Listen() (err error) { | ||
r.recreateTable() | ||
return r.pipe.Listen(r.applyOp) | ||
} | ||
|
||
|
@@ -122,28 +252,12 @@ func (r *Rethinkdb) applyOp(msg *message.Msg) (*message.Msg, error) { | |
return msg, nil | ||
} | ||
|
||
func (r *Rethinkdb) setupClient() (*gorethink.Session, error) { | ||
// set up the clientConfig, we need host:port, username, password, and database name | ||
if r.debug { | ||
fmt.Printf("Connecting to %s\n", r.uri.Host) | ||
} | ||
client, err := gorethink.Connect(gorethink.ConnectOpts{ | ||
Address: r.uri.Host, | ||
MaxIdle: 10, | ||
Timeout: time.Second * 10, | ||
}) | ||
if err != nil { | ||
return nil, fmt.Errorf("unable to connect: %s", err) | ||
} | ||
|
||
func (r *Rethinkdb) recreateTable() { | ||
if r.debug { | ||
fmt.Printf("dropping and creating table '%s' on database '%s'\n", r.table, r.database) | ||
} | ||
gorethink.Db(r.database).TableDrop(r.table).RunWrite(client) | ||
gorethink.Db(r.database).TableCreate(r.table).RunWrite(client) | ||
|
||
client.Use(r.database) | ||
return client, nil | ||
gorethink.Db(r.database).TableDrop(r.table).RunWrite(r.client) | ||
gorethink.Db(r.database).TableCreate(r.table).RunWrite(r.client) | ||
} | ||
|
||
// handleresponse takes the rethink response and turn it into something we can consume elsewhere | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
Source({"name": "rethink1", "namespace": "test.transporter", "tail": true, "debug": true}). | ||
save({"name": "locales", "namespace": "test.transporter"}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -12,6 +12,9 @@ nodes: | |
es: | ||
type: elasticsearch | ||
uri: https://nick:[email protected]:10291/thisgetsignored | ||
locales: | ||
type: elasticsearch | ||
uri: http://localhost:9200/thisgetsignored | ||
timeseries: | ||
type: influx | ||
uri: influxdb://root:root@localhost:8086/compose | ||
|
@@ -32,7 +35,7 @@ nodes: | |
uri: stdout:// | ||
rethink1: | ||
type: rethinkdb | ||
uri: rethink://127.0.0.2:28015/ | ||
uri: rethink://localhost:28015/ | ||
loosefile: | ||
type: file | ||
logtransformer: | ||
|
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This needs to be publicly accessible so that it can be properly "Registered" here as: