Implement pluggable authentication and session support for replicator

Previously replicator used only basic authentication. It was simple and
straightforward. However, with PBKDF2 hashing becoming the default it would be
nice not to do all the password verification work with every single request,
and instead take advantage of session (cookie) based authentication.

This commit implements session based authentication via a plugin mechanism.
The list of available replicator auth modules is configurable. For example:

```
[replicator]
auth_plugins = couch_replicator_auth_session,couch_replicator_auth_basic
```

The plugins will be tried in order. The first one to successfully initialize
will end up being used for that endpoint (source or target).

During the initialization callback, a plugin could decide it cannot be used in
the current context. In that case it signals to be "ignored". The plugin
framework will then skip over it and try to initialize the next on in the list.

`couch_replicator_auth_basic` effectively implements the old behavior. This
plugin should normally be used as a default catch-all at the end of the plugin
list. In some cases, it might be useful to enforce exclusive use of
session-based auth and fail replication jobs if it is not available.

`couch_replicator_auth_session` does most of the work of handling session based
authentication. On initialization, it strips away basic auth credentials from
headers and url to avoid basic auth being used on the server. Then it is in
charge of periodically issuing POST requests to `_session`, updating the
headers of each request with the latest cookie value, and possibly picking up
new session cookie if the server can issue them along with reglar responses.

Currently session based auth plugin is not enabled by default and is an opt-in
feature. That is, users would have to explicitly add the session module to the
list of auth_plugins. In a future, session might be used by default.

As discussed in #1153 this work also removes OAuth 1.0 support. After
server-side support was removed, it had stopped working anyway since the main
oauth app was removed. However, with the plugin framework in place it would be
possible for someone to implement it as a separate module not entangled with
the rest of the replicator code.

Fixes #1153

rel/overlay/etc/default.ini

@@ -430,6 +430,21 @@ ssl_certificate_max_depth = 3
 ; Re-check cluster state at least every cluster_quiet_period seconds
 ; cluster_quiet_period = 60
 
+; List of replicator client authentication plugins to try. Plugins will be
+; tried in order. The first to initialize successfully will be used for that
+; particular endpoint (source or target). Normally couch_replicator_auth_noop
+; would be used at the end of the list as a "catch-all". It doesn't do anything
+; and effectively implements the previous behavior of using basic auth.
+; There are currently two plugins available:
+;   couch_replicator_auth_session - use _session cookie authentication
+;   couch_replicator_auth_noop - use basic authentication (previous default)
+; Currently previous default behavior is still the default. To start using
+; session auth, use this as the list of plugins:
+; `couch_replicator_auth_session,couch_replicator_auth_noop`.
+; In a future release the session plugin might be used by default.
+;auth_plugins = couch_replicator_auth_noop
+
+
 [compaction_daemon]
 ; The delay, in seconds, between each check for which database and view indexes
 ; need to be compacted.

src/couch_replicator/src/couch_replicator_api_wrap.hrl → src/couch_replicator/include/couch_replicator_api_wrap.hrl

@@ -14,7 +14,7 @@
 
 -record(httpdb, {
     url,
-    oauth = nil,
+    auth_props = [],
     headers = [
         {"Accept", "application/json"},
         {"User-Agent", "CouchDB-Replicator/" ++ couch_server:get_version()}
@@ -26,13 +26,6 @@
     httpc_pool = nil,
     http_connections,
     first_error_timestamp = nil,
-    proxy_url
-}).
-
--record(oauth, {
-    consumer_key,
-    token,
-    token_secret,
-    consumer_secret,
-    signature_method
+    proxy_url,
+    auth_context = nil
 }).

src/couch_replicator/src/couch_replicator.erl

@@ -25,7 +25,7 @@
 
 -include_lib("couch/include/couch_db.hrl").
 -include("couch_replicator.hrl").
--include("couch_replicator_api_wrap.hrl").
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
 -include_lib("couch_mrview/include/couch_mrview.hrl").
 -include_lib("mem3/include/mem3.hrl").
 

src/couch_replicator/src/couch_replicator_api_wrap.erl

@@ -142,7 +142,8 @@ db_open(DbName, Options, Create, _CreateParams) ->
         throw({unauthorized, DbName})
     end.
 
-db_close(#httpdb{httpc_pool = Pool}) ->
+db_close(#httpdb{httpc_pool = Pool} = HttpDb) ->
+    couch_replicator_auth:cleanup(HttpDb),
     unlink(Pool),
     ok = couch_replicator_httpc_pool:stop(Pool);
 db_close(DbName) ->
@@ -1009,7 +1010,7 @@ header_value(Key, Headers, Default) ->
 normalize_db(#httpdb{} = HttpDb) ->
     #httpdb{
         url = HttpDb#httpdb.url,
-        oauth = HttpDb#httpdb.oauth,
+        auth_props = lists:sort(HttpDb#httpdb.auth_props),
         headers = lists:keysort(1, HttpDb#httpdb.headers),
         timeout = HttpDb#httpdb.timeout,
         ibrowse_options = lists:keysort(1, HttpDb#httpdb.ibrowse_options),
@@ -1037,7 +1038,7 @@ maybe_append_create_query_params(Db, CreateParams) ->
 normalize_http_db_test() ->
     HttpDb =  #httpdb{
         url = "http://host/db",
-        oauth = #oauth{},
+        auth_props = [{"key", "val"}],
         headers = [{"k2","v2"}, {"k1","v1"}],
         timeout = 30000,
         ibrowse_options = [{k2, v2}, {k1, v1}],

src/couch_replicator/src/couch_replicator_auth.erl

@@ -0,0 +1,99 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_auth).
+
+
+-export([
+    initialize/1,
+    update_headers/2,
+    handle_response/3,
+    cleanup/1
+]).
+
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+
+-type headers() :: [{string(), string()}].
+-type code() :: non_neg_integer().
+
+
+-define(DEFAULT_PLUGINS, "couch_replicator_auth_noop").
+
+
+% Behavior API
+
+-callback initialize(#httpdb{}) -> {ok, #httpdb{}, term()} | ignore.
+
+-callback update_headers(term(), headers()) -> {headers(), term()}.
+
+-callback handle_response(term(), code(), headers()) ->
+    {continue | retry, term()}.
+
+-callback cleanup(term()) -> ok.
+
+
+% Main API
+
+-spec initialize(#httpdb{}) -> {ok, #httpdb{}} | {error, term()}.
+initialize(#httpdb{auth_context = nil} = HttpDb) ->
+    case try_initialize(get_plugin_modules(), HttpDb) of
+        {ok, Mod, HttpDb1, Context} ->
+            {ok, HttpDb1#httpdb{auth_context = {Mod, Context}}};
+        {error, Error} ->
+            {error, Error}
+    end.
+
+
+-spec update_headers(#httpdb{}, headers()) -> {headers(), #httpdb{}}.
+update_headers(#httpdb{auth_context = {Mod, Context}} = HttpDb, Headers) ->
+    {Headers1, Context1} = Mod:update_headers(Context, Headers),
+    {Headers1, HttpDb#httpdb{auth_context = {Mod, Context1}}}.
+
+
+-spec handle_response(#httpdb{}, code(), headers()) ->
+    {continue | retry, term()}.
+handle_response(#httpdb{} = HttpDb, Code, Headers) ->
+    {Mod, Context} = HttpDb#httpdb.auth_context,
+    {Res, Context1} = Mod:handle_response(Context, Code, Headers),
+    {Res, HttpDb#httpdb{auth_context = {Mod, Context1}}}.
+
+
+-spec cleanup(#httpdb{}) -> #httpdb{}.
+cleanup(#httpdb{auth_context = {Module, Context}} = HttpDb) ->
+    ok = Module:cleanup(Context),
+    HttpDb#httpdb{auth_context = nil}.
+
+
+% Private helper functions
+
+-spec get_plugin_modules() -> [atom()].
+get_plugin_modules() ->
+    Plugins1 = config:get("replicator", "auth_plugins", ?DEFAULT_PLUGINS),
+    [list_to_atom(Plugin) || Plugin <- string:tokens(Plugins1, ",")].
+
+
+try_initialize([], _HttpDb) ->
+    {error, no_more_auth_plugins_left_to_try};
+try_initialize([Mod | Modules], HttpDb) ->
+    try Mod:initialize(HttpDb) of
+        {ok, HttpDb1, Context} ->
+            {ok, Mod, HttpDb1, Context};
+        ignore ->
+            try_initialize(Modules, HttpDb);
+        {error, Error} ->
+            {error, Error}
+    catch
+        error:undef ->
+            {error, {could_not_load_plugin_module, Mod}}
+    end.

src/couch_replicator/src/couch_replicator_auth_noop.erl

@@ -0,0 +1,52 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_auth_noop).
+
+
+-behavior(couch_replicator_auth).
+
+
+-export([
+    initialize/1,
+    update_headers/2,
+    handle_response/3,
+    cleanup/1
+]).
+
+
+-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+
+
+-type headers() :: [{string(), string()}].
+-type code() :: non_neg_integer().
+
+
+-spec initialize(#httpdb{}) -> {ok, #httpdb{}, term()} | ignore.
+initialize(#httpdb{} = HttpDb) ->
+    {ok, HttpDb, nil}.
+
+
+-spec update_headers(term(), headers()) -> {headers(), term()}.
+update_headers(Context, Headers) ->
+    {Headers, Context}.
+
+
+-spec handle_response(term(), code(), headers()) ->
+    {continue | retry, term()}.
+handle_response(Context, _Code, _Headers) ->
+    {continue, Context}.
+
+
+-spec cleanup(term()) -> ok.
+cleanup(_Context) ->
+    ok.