Publish Upload Test File (locustio#58)

* change dependencies order in setup.py (locustio#54)

* bugfix when hide edit modal while stopping and edit modal still open (locustio#56)

* rizal new feature upload test file to eb (locustio#53)

* upload test file to deployed eb

* enable user to upload test file to deployed eb

reload list on selector after file uploaded

writing files to slaves

distribute file to slaves and reload the test list

route config json to user fileio

update directories dinamycally

reload after upload, populate directory list

modify interface

add confirmation dialog

* fix non geometric elements

* fix test not reloaded when not in master slave mode, css file and alert message

* erlangga feature create new json keys (locustio#52)

* Revert "erlangga feature create new json keys (locustio#52)" (locustio#57)

This reverts commit acccdf5.

locust/configuration.py

@@ -5,37 +5,7 @@
 from flask import make_response
 
 logger = logging.getLogger(__name__)
-config_path = '/tests/settings/config.json'
-
-def read_file():
-    """
-    Will read the file and return it as a string with tree view.
-    """
-    try:
-        with open((os.environ['PYTHONPATH'].split(os.pathsep))[-1] + config_path, "r") as data_file:
-            data = data_file.read()
-    except Exception as err:
-        logger.info(err)
-        data = "{}"
-    return data
-
-def write_file(string_json):
-    """
-    The `string_json` will overwrite existing configuration. 
-    If the previous configuration doesn't exist, then it will create the file.
-    """
-    status, message = None, None
-    try:
-        with open((os.environ['PYTHONPATH'].split(os.pathsep))[-1] + config_path, "w") as data_file:
-            data_file.write(string_json)
-            status = True
-            message = 'Configuration has been saved'
-            events.master_new_configuration.fire(new_config=string_json)
-    except Exception as err:
-        logger.info(err)
-        status = False
-        message = "Can't save the configuration :" + err
-    return status, message
+CONFIG_PATH = '/tests/settings/config.json'
 
 class ClientConfiguration:
     """
@@ -51,7 +21,7 @@ def read_json(self):
         """
         if self.config_data is None:
             try:
-                with open((os.environ['PYTHONPATH'].split(os.pathsep))[-1] + config_path, "r") as data_file:
+                with open((os.environ['PYTHONPATH'].split(os.pathsep))[-1] + CONFIG_PATH, "r") as data_file:
                     self.config_data = json.load(data_file)
             except Exception as err:
                 logger.info(err)

locust/events.py

@@ -135,3 +135,8 @@ def fire(self, **kwargs):
 """
 *master_new_configuration* is fired when new configuration saved from master
 """
+
+master_new_file_uploaded = EventHook()
+"""
+*master_new_file_uploaded* is fired when user upload a new test file
+"""

locust/fileio.py

@@ -0,0 +1,30 @@
+import logging
+import os
+import events
+import werkzeug
+
+logger = logging.getLogger(__name__)
+
+def read(uploaded_file_path):
+    try:
+        with open(os_path()+uploaded_file_path, "r") as data_file:
+            data = data_file.read()
+    except IOError as err:
+        logger.info("error read: " + err.strerror) 
+        data = None
+    return data
+
+def write(file_path, file_content):
+    try:
+        with open(os_path()+file_path, "w") as data_file:
+            data_file.write(file_content)
+            status = True
+            message = 'File has been saved'
+    except IOError as err:
+        logger.info("error write: " + err.strerror )
+        status = False
+        message = "Can't overwrite protected file. Please rename your test file. i.e : home_production_dilan.py"
+    return status, message
+
+def os_path():
+    return os.environ['PYTHONPATH'].split(os.pathsep)[-1]+os.sep

locust/main.py

@@ -10,6 +10,8 @@
 import logging
 import socket
 import time
+import tests_loader
+import fileio
 from optparse import OptionParser
 
 from . import web
@@ -261,137 +263,6 @@ def parse_options():
     opts, args = parser.parse_args()
     return parser, opts, args
 
-
-def _is_package(path):
-    """
-    Is the given path a Python package?
-    """
-    return (
-        os.path.isdir(path)
-        and os.path.exists(os.path.join(path, '__init__.py'))
-    )
-
-
-def find_locustfile(locustfile):
-    """
-    Attempt to locate a locustfile, either explicitly or by searching parent dirs.
-    """
-    # Obtain env value
-    names = [locustfile]
-    # Create .py version if necessary
-    if not names[0].endswith('.py'):
-        names += [names[0] + '.py']
-    # Does the name contain path elements?
-    if os.path.dirname(names[0]):
-        # If so, expand home-directory markers and test for existence
-        for name in names:
-            expanded = os.path.expanduser(name)
-            if os.path.exists(expanded):
-                if name.endswith('.py') or _is_package(expanded):
-                    return os.path.abspath(expanded)
-    else:
-        # Otherwise, start in cwd and work downwards towards filesystem root
-        path = '.'
-        # Stop before falling off root of filesystem (should be platform
-        # agnostic)
-        while os.path.split(os.path.abspath(path))[1]:
-            for name in names:
-                joined = os.path.join(path, name)
-                if os.path.exists(joined):
-                    if name.endswith('.py') or _is_package(joined):
-                        return os.path.abspath(joined)
-            path = os.path.join('..', path)
-    # Implicit 'return None' if nothing was found
-
-
-def is_locust(tup):
-    """
-    Takes (name, object) tuple, returns True if it's a public Locust subclass.
-    """
-    name, item = tup
-    return bool(
-        inspect.isclass(item)
-        and issubclass(item, Locust)
-        and hasattr(item, "task_set")
-        and getattr(item, "task_set")
-        and not name.startswith('_')
-    )
-
-def truncate_path(path):
-    # split path which comes from command on terminal
-    splitted_path = os.path.normpath(path).split(os.path.sep)
-
-    count = 0
-    for i in reversed(xrange(len(splitted_path))):
-        if count < 3 and splitted_path[i]:
-            if count == 0:
-                final_path = splitted_path[i]
-            elif count == 2:
-                final_path = os.path.join("...", splitted_path[i], final_path)
-            else:
-                final_path = os.path.join(splitted_path[i], final_path)
-            count += 1
-        else:
-            break
-    return final_path
-
-def load_locustfile(path):
-    """
-    Import given locustfile path and return (docstring, callables).
-
-    Specifically, the locustfile's ``__doc__`` attribute (a string) and a
-    dictionary of ``{'name': callable}`` containing all callables which pass
-    the "is a Locust" test.
-    """
-    # Get directory and locustfile name
-    directory, locustfile = os.path.split(path)
-    # If the directory isn't in the PYTHONPATH, add it so our import will work
-    added_to_path = False
-    index = None
-    if directory not in sys.path:
-        sys.path.insert(0, directory)
-        added_to_path = True
-    # If the directory IS in the PYTHONPATH, move it to the front temporarily,
-    # otherwise other locustfiles -- like Locusts's own -- may scoop the intended
-    # one.
-    else:
-        i = sys.path.index(directory)
-        if i != 0:
-            # Store index for later restoration
-            index = i
-            # Add to front, then remove from original position
-            sys.path.insert(0, directory)
-            del sys.path[i + 1]
-    # Perform the import (trimming off the .py)
-    imported = imp.load_source(os.path.splitext(locustfile)[0], path)
-    # Remove directory from path if we added it ourselves (just to be neat)
-    if added_to_path:
-        del sys.path[0]
-    # Put back in original index if we moved it
-    if index is not None:
-        sys.path.insert(index + 1, directory)
-        del sys.path[0]
-    # Return our two-tuple
-    locusts = dict(filter(is_locust, vars(imported).items()))
-
-    # truncate the fullpath
-    final_path = truncate_path(path)
-
-    return {final_path: locusts}
-
-def collect_locustfiles(path):
-    collected = dict()
-
-    for root, dirs, files in os.walk(path):
-        if files:
-            for file_ in files:
-                if file_.endswith('.py') and not file_.endswith('__init__.py'):
-                    fullpath = os.path.abspath(os.path.join(root, file_))
-                    loaded = load_locustfile(fullpath)
-                    if loaded:
-                        collected.update(loaded)
-    return collected
-
 def main():
     parser, options, arguments = parse_options()
 
@@ -404,9 +275,9 @@ def main():
         sys.exit(0)
 
     if os.path.isdir(options.locustfile):
-        all_locustfiles = collect_locustfiles(options.locustfile)
+        all_locustfiles = tests_loader.collect_locustfiles(options.locustfile)
     else:
-        locustfile = find_locustfile(options.locustfile)
+        locustfile = tests_loader.find_locustfile(options.locustfile)
 
         if not locustfile:
             logger.error("Could not find any locustfile! Ensure file ends in '.py' and see --help for available options.")
@@ -417,7 +288,7 @@ def main():
             sys.exit(1)
 
         # docstring, all_locustsfiles = load_locustfile(locustfile)
-        all_locustfiles = load_locustfile(locustfile)
+        all_locustfiles = tests_loader.load_locustfile(locustfile)
 
     # Use the first locustfile for the default locusts
     locusts = all_locustfiles.values()[0]

locust/runners.py

@@ -18,6 +18,9 @@
 
 from .rpc import rpc, Message
 
+import fileio
+import tests_loader
+
 logger = logging.getLogger(__name__)
 
 # global locust runner singleton
@@ -203,6 +206,9 @@ def log_exception(self, node_id, msg, formatted_tb):
         row["count"] += 1
         row["nodes"].add(node_id)
         self.exceptions[key] = row
+
+    def reload_tests(self):
+        self.available_locustfiles = tests_loader.load(self.options.locustfile)
 
 class LocalLocustRunner(LocustRunner):
     def __init__(self, locust_classes, options, available_locustfiles=None):
@@ -254,7 +260,6 @@ def hatching(self):
             @property
             def running(self):
                 return self.get_by_state(STATE_RUNNING)
-
         self.clients = SlaveNodesDict()
         self.server = rpc.Server(self.master_bind_host, self.master_bind_port)
         self.greenlet = Group()
@@ -288,6 +293,13 @@ def on_master_new_configuration(new_config):
                 self.server.send(Message("config", data, None))
         events.master_new_configuration += on_master_new_configuration
 
+        def on_master_new_file_uploaded(new_file):
+            logger.info("master has been received a new test file, slaves should update theirs too")
+            self.reload_tests()
+            for client in six.itervalues(self.clients):
+                self.server.send(Message("python_file", new_file, None))
+        events.master_new_file_uploaded += on_master_new_file_uploaded
+
     @property
     def user_count(self):
         return sum([c.user_count for c in six.itervalues(self.clients)])
@@ -432,7 +444,15 @@ def worker(self):
                 self.locust_classes = self.available_locustfiles[msg.data].values()
             elif msg.type == "config":
                 logger.info("Got new config from master, updating this slave config")
-                configuration.write_file(msg.data['config'])
+                fileio.write(configuration.CONFIG_PATH, msg.data['config'])
+                events.master_new_configuration.fire(new_config=msg.data['config'])
+            elif msg.type == "python_file":
+                logger.info("Uploaded test file from master detected, writing now")
+                new_file = msg.data
+                upload_status,upload_message = fileio.write(new_file['full_path'], new_file['content'])
+                if upload_status is False :
+                    logger.info("error while creating new file: " + upload_message)
+                self.reload_tests()
 
 
     def stats_reporter(self):