Add feature to allow ceryx to be used as data by-pass proxy

README.md

@@ -103,7 +103,16 @@ docker-compose exec api bin/populate-api
 ```
 curl -H "Content-Type: application/json" \
      -X POST \
-     -d '{"source":"publicly.accessible.domain","target":"http://service.internal:8000"}' \
+     -d '{"source": "any-valid-hostname", "target": "http://service.internal:8000"}' \
+     http://ceryx-api-host/api/routes
+```
+
+A route may also have request parameters in the target.
+
+```
+curl -H "Content-Type: application/json" \
+     -X POST \
+     -d '{"source": "any-valid-hostname", "target": "http://service.internal:8000?foo=bar&x=y"}' \
      http://ceryx-api-host/api/routes
 ```
 
@@ -112,16 +121,16 @@ curl -H "Content-Type: application/json" \
 ```
 curl -H "Content-Type: application/json" \
      -X PUT \
-     -d '{"source":"publicly.accessible.domain","target":"http://another-service.internal:8000"}' \
-     http://ceryx-api-host/api/routes/publicly.accessible.domain
+     -d '{"source": "any-valid-hostname", "target": "http://another-service.internal:8000"}' \
+     http://ceryx-api-host/api/routes/any-valid-hostname
 ```
 
 ### Delete a route from Ceryx
 
 ```
 curl -H "Content-Type: application/json" \
      -X DELETE \
-     http://ceryx-api-host/api/routes/publicly.accessible.domain
+     http://ceryx-api-host/api/routes/any-valid-hostname
 ```
 
 ### Enforce HTTPS
@@ -131,20 +140,42 @@ You can enforce redirection from HTTP to HTTPS for any host you would like.
 ```
 curl -H "Content-Type: application/json" \
      -X POST \
-     -d '{"source":"publicly.accessible.domain","target":"http://service.internal:8000", "settings": {"enforce_https": true}}' \
+     -d '{"source": "www.sourcelair.com", "target":"http://service.internal:8000", "settings": {"enforce_https": true}}' \
      http://ceryx-api-host/api/routes
 ```
 
 The above functionality works in `PUT` update requests as well.
 
 ### Redirect to target, instead of proxying
 
-Instead of proxying the request to the targetm you can prompt the client to redirect the request there itself.
+Instead of proxying the request to the target, you can prompt the client to redirect the request there itself.
+
+```
+curl -H "Content-Type: application/json" \
+     -X POST \
+     -d '{"source": "sourcelair.com", "target":"https://www.sourcelair.com", "settings": {"mode": "redirect"}}' \
+     http://ceryx-api-host/api/routes
+```
+
+### Include additional headers (e.g. `Authorization`) for target connection
+
+If the route should be authorized behind Ceryx you may deploy an `Authorization` header with the route.
+
+```
+curl -H "Content-Type: application/json" \
+     -X POST \
+     -d '{"source":"sourcelair.com", "target":"https://www.sourcelair.com", "settings": {"headers": {"authorization": "Bearer ..."}}}' \
+     http://ceryx-api-host/api/routes
+```
+
+### Give routes a TTL
+
+You can provide a TTL (in seconds) for your routes after which they are removed from Ceryx.
 
 ```
 curl -H "Content-Type: application/json" \
      -X POST \
-     -d '{"source":"sourcelair.com","target":"https://www.sourcelair.com", "settings": {"mode": "redirect"}}' \
+     -d '{"source":"sourcelair.com", "target": "https://www.sourcelair.com", "settings": {"ttl": 20}}' \
      http://ceryx-api-host/api/routes
 ```
 
@@ -158,6 +189,7 @@ Ceryx has proven to be extremely reliable in production systems, handling tens o
 
 - [**SourceLair**](https://www.sourcelair.com/): In-browser IDE for web applications, made publicly accessible via development web servers powered by Ceryx.
 - [**Stolos**](http://stolos.io/): Managed Docker development environments for enterprises.
+- [**othermo**](https://www.othermo.de): Industry 4.0 for heating plants and municipal utilities, using Ceryx to implement the [Data By-Pass Pattern](https://www.eclipse.org/ditto/advanced-data-by-pass.html) with [Eclipse Ditto](https://www.eclipse.org/ditto).
 
 Do you use Ceryx in production as well? Please [open a Pull Request](https://github.com/sourcelair/ceryx/pulls) to include it here. We would love to have it in our list.
 

api/api.py

@@ -8,7 +8,7 @@
 client = RedisClient.from_config()
 
 
-@api.route(default=True)
+@api.route("/", default=True)
 def default(req, resp):
     if not req.url.path.endswith("/"):
         api.redirect(resp, f"{req.url.path}/")

api/ceryx/db.py

@@ -39,71 +39,72 @@ def _route_key(self, source):
     def _settings_key(self, source):
         return self._prefixed_key(f"settings:{source}")
 
-    def _delete_target(self, host):
-        key = self._route_key(host)
+    def _delete_target(self, source):
+        key = self._route_key(source)
         self.client.delete(key)
 
-    def _delete_settings(self, host):
-        key = self._settings_key(host)
+    def _delete_settings(self, source):
+        key = self._settings_key(source)
         self.client.delete(key)
 
-    def _lookup_target(self, host, raise_exception=False):
-        key = self._route_key(host)
+    def _lookup_target(self, source, raise_exception=False):
+        key = self._route_key(source)
         target = self.client.get(key)
-        
+
         if target is None and raise_exception:
             raise exceptions.NotFound("Route not found.")
 
         return target
 
-    def _lookup_settings(self, host):
-        key = self._settings_key(host)
+    def _lookup_settings(self, source):
+        key = self._settings_key(source)
         return self.client.hgetall(key)
 
-    def lookup_hosts(self, pattern="*"):
+    def lookup_sources(self, pattern="*"):
         lookup_pattern = self._route_key(pattern)
         left_padding = len(lookup_pattern) - 1
         keys = self.client.keys(lookup_pattern)
         return [_str(key)[left_padding:] for key in keys]
 
-    def _set_target(self, host, target):
-        key = self._route_key(host)
-        self.client.set(key, target)
+    def _set_target(self, source, target, ttl=None):
+        key = self._route_key(source)
+        self.client.set(key, target, ex=ttl)
 
-    def _set_settings(self, host, settings):
-        key = self._settings_key(host)
+    def _set_settings(self, source, settings):
+        self._delete_settings(source)
+        key = self._settings_key(source)
         self.client.hmset(key, settings)
 
     def _set_route(self, route: schemas.Route):
         redis_data = route.to_redis()
-        self._set_target(route.source, redis_data["target"])
+        self._set_target(route.source, redis_data["target"], route.settings.get("ttl"))
         self._set_settings(route.source, redis_data["settings"])
         return route
 
-    def get_route(self, host):
-        target = self._lookup_target(host, raise_exception=True)
-        settings = self._lookup_settings(host)
+    def get_route(self, source):
+        target = self._lookup_target(source, raise_exception=True)
+        settings = self._lookup_settings(source)
         route = schemas.Route.from_redis({
-            "source": host,
+            "source": source,
             "target": target,
             "settings": settings
         })
         return route
 
     def list_routes(self):
-        hosts = self.lookup_hosts()
-        routes = [self.get_route(host) for host in hosts]
+        sources = self.lookup_sources()
+        routes = [self.get_route(source) for source in sources]
         return routes
 
     def create_route(self, data: dict):
         route = schemas.Route.validate(data)
         return self._set_route(route)
 
-    def update_route(self, host: str, data: dict):
-        data["source"] = host
+    def update_route(self, source: str, data: dict):
+        data["source"] = source
         route = schemas.Route.validate(data)
         return self._set_route(route)
 
-    def delete_route(self, host: str):
-        self._delete_target(host)
-        self._delete_settings(host)
+    def delete_route(self, source: str):
+        self._delete_target(source)
+        self._delete_settings(source)

api/ceryx/schemas.py

@@ -1,6 +1,6 @@
 import re
 import typesystem
-
+import json
 
 def ensure_protocol(url):
     starts_with_protocol = r"^https?://"
@@ -15,6 +15,22 @@ def redis_to_boolean(value):
     return True if value == "1" else False
 
 
+def object_to_redis(value: object):
+    return json.dumps(value)
+
+
+def redis_to_object(value):
+    return json.loads(value)
+
+
+def integer_to_redis(value: int):
+    return str(value)
+
+
+def redis_to_integer(value):
+    return int(value)
+
+
 def ensure_string(value):
     redis_value = (
         None if value is None
@@ -30,6 +46,12 @@ def value_to_redis(field, value):
     if isinstance(field, typesystem.Reference):
         return field.target.validate(value).to_redis()
 
+    if isinstance(field, typesystem.Object):
+        return object_to_redis(value)
+
+    if isinstance(field, typesystem.Integer):
+        return integer_to_redis(value)
+
     return ensure_string(value)
 
 
@@ -40,6 +62,12 @@ def redis_to_value(field, redis_value):
     if isinstance(field, typesystem.Reference):
         return field.target.from_redis(redis_value)
 
+    if isinstance(field, typesystem.Object):
+        return redis_to_object(redis_value)
+
+    if isinstance(field, typesystem.Integer):
+        return redis_to_integer(redis_value)
+
     return ensure_string(redis_value)
 
 
@@ -69,6 +97,8 @@ class Settings(BaseSchema):
         ),
         default="proxy",
     )
+    headers = typesystem.Object(default={}, properties=typesystem.String(max_length=100))
+    ttl = typesystem.Integer(allow_null=True)
     certificate_path = typesystem.String(allow_null=True)
     key_path = typesystem.String(allow_null=True)
 

ceryx/nginx/lualib/ceryx/routes.lua

@@ -1,4 +1,5 @@
 local redis = require "ceryx.redis"
+local cjson = require "cjson"
 
 local exports = {}
 
@@ -16,7 +17,7 @@ end
 
 function getTargetForSource(source, redisClient)
     -- Construct Redis key and then
-    -- try to get target for host
+    -- try to get target for source
     local key = getRouteKeyForSource(source)
     local target, _ = redisClient:get(key)
 
@@ -49,6 +50,18 @@ function getModeForSource(source, redisClient)
     return mode
 end
 
+function getHeadersForSource(source, redisClient)
+    ngx.log(ngx.DEBUG, "Get routing headers for " .. source .. ".")
+    local settings_key = getSettingsKeyForSource(source)
+    local headers, _ = cjson.decode(redisClient:hget(settings_key, "headers"))
+
+    if headers == ngx.null or not headers then
+        headers = {}
+    end
+
+    return headers
+end
+
 function getRouteForSource(source)
     local _
     local route = {}
@@ -69,11 +82,12 @@ function getRouteForSource(source)
         if targetIsInValid(route.target) then
             return nil
         end
-        cache:set(host, res, 5)
+        cache:set(source, res, 5)
         ngx.log(ngx.DEBUG, "Caching from " .. source .. " to " .. route.target .. " for 5 seconds.")
     end
 
     route.mode = getModeForSource(source, redisClient)
+    route.headers = getHeadersForSource(source, redisClient)
 
     return route
 end

ceryx/nginx/lualib/router.lua

@@ -5,37 +5,46 @@ local utils = require "ceryx.utils"
 local redisClient = redis:client()
 
 local host = ngx.var.host
+local request_uri = ngx.var.request_uri:sub(2)
 local cache = ngx.shared.ceryx
 
 local is_not_https = (ngx.var.scheme ~= "https")
 
-function formatTarget(target)
+function formatTargetHostSource(target)
     target = utils.ensure_protocol(target)
     target = utils.ensure_no_trailing_slash(target)
-
     return target .. ngx.var.request_uri
 end
 
-function redirect(source, target)
-    ngx.log(ngx.INFO, "Redirecting request for " .. source .. " to " .. target .. ".")
+function formatTargetRequestUriSource(target)
+    target = utils.ensure_protocol(target)
+    return target
+end
+
+function redirect(source, target, headers)
+    ngx.log(ngx.INFO, "Redirecting request for " .. source)
+    for k,v in pairs(headers) do
+        ngx.headers[k] = v
+    end
     return ngx.redirect(target, ngx.HTTP_MOVED_PERMANENTLY)
 end
 
-function proxy(source, target)
+function proxy(source, target, headers)
     ngx.var.target = target
-    ngx.log(ngx.INFO, "Proxying request for " .. source .. " to " .. target .. ".")
+    for k,v in pairs(headers) do
+        ngx.req.set_header(k, v)
+    end
+    ngx.log(ngx.INFO, "Proxying request for " .. source)
 end
 
-function routeRequest(source, target, mode)
+function routeRequest(source, target, mode, headers)
     ngx.log(ngx.DEBUG, "Received " .. mode .. " routing request from " .. source .. " to " .. target)
 
-    target = formatTarget(target)
-
     if mode == "redirect" then
-        return redirect(source, target)
+        return redirect(source, target, headers)
     end
 
-    return proxy(source, target)
+    return proxy(source, target, headers)
 end
 
 if is_not_https then
@@ -53,13 +62,19 @@ if is_not_https then
     end
 end
 
-ngx.log(ngx.INFO, "HOST " .. host)
+ngx.log(ngx.INFO, "Try host route for " .. host)
 local route = routes.getRouteForSource(host)
 
-if route == nil then
-    ngx.log(ngx.INFO, "No $wildcard target configured for fallback. Exiting with Bad Gateway.")
-    return ngx.exit(ngx.HTTP_SERVICE_UNAVAILABLE)
+if route ~= nil then
+    return routeRequest(host, formatTargetHostSource(route.target), route.mode, route.headers)
+end
+
+ngx.log(ngx.INFO, "Try request_uri route for " .. request_uri)
+route = routes.getRouteForSource(request_uri)
+
+if route ~= nil then
+    return routeRequest(request_uri, formatTargetRequestUriSource(route.target), route.mode, route.headers)
 end
 
--- Save found key to local cache for 5 seconds
-routeRequest(host, route.target, route.mode)
+ngx.log(ngx.INFO, "No $wildcard target configured for fallback. Exiting with Bad Gateway.")
+ngx.exit(ngx.HTTP_SERVICE_UNAVAILABLE)