Dispatch Routing from yaml

AdminServer/appscale/admin/__init__.py

@@ -67,7 +67,8 @@
   DeleteServiceOperation,
   CreateVersionOperation,
   DeleteVersionOperation,
-  UpdateVersionOperation
+  UpdateVersionOperation,
+  UpdateApplicationOperation
 )
 from .operations_cache import OperationsCache
 from .push_worker_manager import GlobalPushWorkerManager
@@ -221,6 +222,83 @@ class LifecycleState(object):
   DELETE_IN_PROGRESS = 'DELETE_IN_PROGRESS'
 
 
+class AppsHandler(BaseHandler):
+  """ Manages applications. """
+  def initialize(self, ua_client, zk_client):
+    """ Defines required resources to handle requests.
+
+    Args:
+      ua_client: A UAClient.
+      zk_client: A KazooClient.
+    """
+    self.ua_client = ua_client
+    self.zk_client = zk_client
+
+
+  @gen.coroutine
+  def patch(self, name):
+    """ Updates an Application. Currently this is only supported for the
+    dispatch rules.
+
+    Args:
+      name: A string specifying a project ID.
+    """
+    self.authenticate(name, self.ua_client)
+
+    if name in constants.IMMUTABLE_PROJECTS:
+      raise CustomHTTPError(HTTPCodes.BAD_REQUEST,
+                            message='{} cannot be updated'.format(name))
+
+    update_mask = self.get_argument('updateMask', None)
+    if not update_mask:
+      message = 'At least one field must be specified for this operation.'
+      raise CustomHTTPError(HTTPCodes.BAD_REQUEST, message=message)
+
+    desired_fields = update_mask.split(',')
+    supported_fields = {'dispatchRules'}
+    for field in desired_fields:
+      if field not in supported_fields:
+        message = ('This operation is only supported on the following '
+                   'field(s): [{}]'.format(', '.join(supported_fields)))
+        raise CustomHTTPError(HTTPCodes.BAD_REQUEST, message=message)
+
+    project_node = '/'.join(['/appscale', 'projects', name])
+    services_node = '/'.join([project_node, 'services'])
+    if not self.zk_client.exists(project_node):
+      raise CustomHTTPError(HTTPCodes.NOT_FOUND,
+                            message='Project does not exist')
+
+    try:
+      service_ids = self.zk_client.get_children(services_node)
+    except NoNodeError:
+      raise CustomHTTPError(HTTPCodes.INTERNAL_ERROR,
+                            message='Services node not found for project')
+    payload = json.loads(self.request.body)
+    dispatch_rules = utils.routing_rules_from_dict(payload=payload,
+                                                   project_id=name,
+                                                   services=service_ids)
+
+    dispatch_node = '/'.join(['/appscale', 'projects', name, 'dispatch'])
+
+    try:
+      self.zk_client.set(dispatch_node, json.dumps(dispatch_rules))
+    except NoNodeError:
+      try:
+        self.zk_client.create(dispatch_node, json.dumps(dispatch_rules))
+      except NoNodeError:
+        raise CustomHTTPError(HTTPCodes.NOT_FOUND,
+                              message='{} not found'.format(name))
+
+    logger.info('Updated dispatch for {}'.format(name))
+    # TODO: add verification for dispatchRules being applied. For now,
+    # assume the controller picks it up instantly.
+    patch_operation = UpdateApplicationOperation(name)
+    patch_operation.finish(dispatch_rules)
+    operations[patch_operation.id] = patch_operation
+    self.write(json_encode(patch_operation.rest_repr()))
+
+
+
 class BaseVersionHandler(BaseHandler):
 
   def get_version(self, project_id, service_id, version_id):
@@ -1392,6 +1470,8 @@ def main():
      {'acc': acc, 'ua_client': ua_client, 'zk_client': zk_client,
       'version_update_lock': version_update_lock, 'thread_pool': thread_pool,
       'controller_state': controller_state}),
+    ('/v1/apps/([^/]*)', AppsHandler,
+     {'ua_client': ua_client, 'zk_client': zk_client}),
     ('/v1/apps/([^/]*)/operations/([a-z0-9-]+)', OperationsHandler,
      {'ua_client': ua_client}),
     ('/api/cron/update', UpdateCronHandler,

AdminServer/appscale/admin/constants.py

@@ -42,6 +42,7 @@ class Methods(object):
   CREATE_VERSION = 'google.appengine.v1.Versions.CreateVersion'
   DELETE_VERSION = 'google.appengine.v1.Versions.DeleteVersion'
   UPDATE_VERSION = 'google.appengine.v1.Versions.UpdateVersion'
+  UPDATE_APPLICATION = 'google.appengine.v1.Applications.UpdateApplication'
 
 
 class OperationTimeout(Exception):
@@ -74,6 +75,11 @@ class InvalidQueueConfiguration(Exception):
   pass
 
 
+class InvalidDispatchConfiguration(Exception):
+  """ Indicates that the dispatch rule is not valid. """
+  pass
+
+
 class ServingStatus(object):
   """ The possible serving states for a project or version. """
   SERVING = 'SERVING'
@@ -86,6 +92,7 @@ class Types(object):
   EMPTY = 'type.googleapis.com/google.protobuf.Empty'
   OPERATION_METADATA = 'type.googleapis.com/google.appengine.v1.OperationMetadataV1'
   VERSION = 'type.googleapis.com/google.appengine.v1.Version'
+  APPLICATION = 'type.googleapis.com/google.appengine.v1.Application'
 
 
 # The parent directory for source code extraction.
@@ -185,6 +192,45 @@ class Types(object):
 # A regex rule for validating targets, will not match instance.version.module.
 TQ_TARGET_REGEX = re.compile(r'^([a-zA-Z0-9\-]+[\.]?[a-zA-Z0-9\-]*)$')
 
+# A set of regex rules to validate dispatch domains.
+DISPATCH_DOMAIN_REGEX_SINGLE_ASTERISK = re.compile(r'^\*$')
+DISPATCH_DOMAIN_REGEX_ASTERISKS = re.compile(r'\*')
+DISPATCH_DOMAIN_REGEX_ASTERISK_DOT = re.compile(r'\*\.')
+_URL_SPLITTER_RE = re.compile(r'^([^/]+)(/.*)$')
+
+# Regular expression for a hostname based on
+# http://tools.ietf.org/html/rfc1123.
+#
+# This pattern is more restrictive than the RFC because it only accepts
+# lower case letters.
+_URL_HOST_EXACT_PATTERN_RE = re.compile(r"""
+# 0 or more . terminated hostname segments (may not start or end in -).
+^([a-z0-9]([a-z0-9\-]*[a-z0-9])*\.)*
+# followed by a host name segment.
+([a-z0-9]([a-z0-9\-]*[a-z0-9])*)$""", re.VERBOSE)
+
+_URL_IP_V4_ADDR_RE = re.compile(r"""
+#4 1-3 digit numbers separated by .
+^([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$""", re.VERBOSE)
+
+# Regular expression for a prefix of a hostname based on
+# http://tools.ietf.org/html/rfc1123. Restricted to lower case letters.
+_URL_HOST_SUFFIX_PATTERN_RE = re.compile(r"""
+# Single star or
+^([*]|
+# Host prefix with no .,  Ex '*-a' or
+[*][a-z0-9\-]*[a-z0-9]|
+# Host prefix with ., Ex '*-a.b-c.d'
+[*](\.|[a-z0-9\-]*[a-z0-9]\.)([a-z0-9]([a-z0-9\-]*[a-z0-9])*\.)*
+([a-z0-9]([a-z0-9\-]*[a-z0-9])*))$
+""", re.VERBOSE)
+
+# A set of regex rules to validate dispatch paths.
+DISPATCH_PATH_REGEX = re.compile(r'/[0-9a-z/]*[*]?$')
+
+# A regex rule to help make dispatch urls nginx friendly.
+NGINX_DISPATCH_REGEX = re.compile(r'\w*(?<!\.)\*')
+
 REQUIRED_PULL_QUEUE_FIELDS = ['name', 'mode']
 
 REQUIRED_PUSH_QUEUE_FIELDS = ['name', 'rate']

AdminServer/appscale/admin/operation.py

@@ -161,3 +161,35 @@ def __init__(self, project_id, service_id, version):
     super(UpdateVersionOperation, self).__init__(
       project_id, service_id, version)
     self.method = Methods.UPDATE_VERSION
+
+
+class UpdateApplicationOperation(Operation):
+  """ A container that keeps track of CreateVersion operations. """
+  def __init__(self, project_id):
+    """ Creates a new CreateVersionOperation.
+
+    Args:
+      project_id: A string specifying a project ID.
+    """
+    super(UpdateApplicationOperation, self).__init__(project_id)
+    self.method = Methods.UPDATE_APPLICATION
+
+  def finish(self, dispatchRules):
+    """ Marks the operation as completed.
+
+    Args:
+      dispatchRules: A list of the dispatch rules that were applied to the
+        application object.
+    """
+    self.response = {
+      '@type': Types.VERSION,
+      'name': 'apps/{}'.format(self.project_id),
+      'id': self.project_id,
+      'dispatchRules': dispatchRules,
+      'servingStatus': ServingStatus.SERVING,
+      # TODO: add other fields to response for UpdateApplication
+      # "authDomain", "locationId", "codeBucket", "defaultHostname",
+      # "defaultBucket", "gcrDomain"
+    }
+
+    self.done = True

AdminServer/appscale/admin/utils.py

@@ -14,16 +14,26 @@
 from kazoo.exceptions import NoNodeError
 
 from .constants import (
+  _URL_HOST_EXACT_PATTERN_RE,
+  _URL_HOST_SUFFIX_PATTERN_RE,
+  _URL_IP_V4_ADDR_RE,
   AUTO_HTTP_PORTS,
   AUTO_HTTPS_PORTS,
   CustomHTTPError,
+  DEFAULT_VERSION,
+  DISPATCH_PATH_REGEX,
+  DISPATCH_DOMAIN_REGEX_SINGLE_ASTERISK,
+  DISPATCH_DOMAIN_REGEX_ASTERISKS,
+  DISPATCH_DOMAIN_REGEX_ASTERISK_DOT,
   HAPROXY_PORTS,
   GO,
   InvalidCronConfiguration,
+  InvalidDispatchConfiguration,
   InvalidQueueConfiguration,
   InvalidSource,
   JAVA,
   JAVA8,
+  NGINX_DISPATCH_REGEX,
   REQUIRED_PULL_QUEUE_FIELDS,
   REQUIRED_PUSH_QUEUE_FIELDS,
   SOURCES_DIRECTORY,
@@ -581,3 +591,72 @@ def _constant_time_compare(val_a, val_b):
   constant_time_compare = hmac.compare_digest
 else:
   constant_time_compare = _constant_time_compare
+
+
+def validate_routing_rule(rule, services):
+  """
+  domain: string. Domain name to match against. The wildcard "*" is supported
+    if specified before a period: "*.". Defaults to matching all domains: "*".
+  path: string. Pathname within the host. Must start with a "/". A single "*"
+    can be included at the end of the path. The sum of the lengths of the
+    domain and path may not exceed 100 characters.
+  service: string. Resource ID of a service in this application that should
+    serve the matched request. The service must already exist. Example: default.
+  Tip: You can include glob patterns like the `*` wildcard character in the
+   `url` element; however, those patterns can be used only before the host name
+   and at the end of the URL path.
+
+  """
+  service = rule['service']
+  domain = rule['domain']
+  path = rule['path']
+  if service not in services:
+    raise InvalidDispatchConfiguration('Service does not exist.')
+  if domain.startswith('*'):
+    if not _URL_HOST_SUFFIX_PATTERN_RE.match(domain):
+      raise InvalidDispatchConfiguration(
+        'Invalid host pattern {}'.format(domain))
+  else:
+    if not _URL_HOST_EXACT_PATTERN_RE.match(domain):
+      raise InvalidDispatchConfiguration(
+          'Invalid host pattern {}'.format(domain))
+    matcher = _URL_IP_V4_ADDR_RE.match(domain)
+    if matcher and sum(1 for x in matcher.groups() if int(x) <= 255) == 4:
+      raise InvalidDispatchConfiguration(
+          'Host may not match an ipv4 address {}'.format(domain))
+
+  if not DISPATCH_PATH_REGEX.match(path):
+    raise InvalidDispatchConfiguration('Invalid path pattern {}.'.format(path))
+
+  if not DISPATCH_DOMAIN_REGEX_SINGLE_ASTERISK.match(domain):
+    asterisks = DISPATCH_DOMAIN_REGEX_ASTERISKS.findall(domain)
+    asterisk_dot = DISPATCH_DOMAIN_REGEX_ASTERISK_DOT.findall(domain)
+    if len(asterisks) != len(asterisk_dot):
+      raise InvalidDispatchConfiguration(
+          'Invalid host pattern {}'.format(domain))
+
+  if len(domain) + len(path) > 100:
+    raise InvalidDispatchConfiguration('URL over 100 characters.')
+
+
+def routing_rules_from_dict(payload, project_id, services):
+  try:
+    given_routing_rules = payload['dispatchRules']
+  except KeyError:
+    raise InvalidQueueConfiguration('Payload must contain dispatchRules')
+
+  rules = []
+  for rule in given_routing_rules:
+    validate_routing_rule(rule, services)
+    rules.append(convert_to_nginx_friendly_rule(rule, project_id))
+
+  return rules
+
+def convert_to_nginx_friendly_rule(rule, project_id):
+  # Domain defaults to '*'
+  domain = rule['domain'] or '*'
+  url = '{}{}'.format(domain, rule['path'])
+  nginx_url = NGINX_DISPATCH_REGEX.sub('.*', url)
+  nginx_service_backend = 'gae_{}_{}_{}'.format(project_id, rule['service'],
+                                                DEFAULT_VERSION)
+  return {'url': nginx_url, 'service': nginx_service_backend}