Merge branch 'master' into vm_bench_tests

.gitignore

@@ -7,6 +7,7 @@ target/
 # IntelliJ
 .idea
 .vscode
+*.iml
 
 # MacOS folder metadata
 .DS_Store

.travis.yml

@@ -4,6 +4,12 @@ matrix:
       scala: 2.12.7
       jdk: oraclejdk8
 
+      services:
+        - docker
+      install:
+        - docker pull fluencelabs/solver:2018-dec-demo
+        - docker pull fluencelabs/node:2018-dec-demo
+
       # These directories are cached to S3 at the end of the build
       cache:
         directories:
@@ -47,15 +53,16 @@ matrix:
 
       script:
         - cargo fmt --all -- --check -v
-        - cargo build -v
-        - cargo doc -v
-        - cargo test -v
+        - cargo build -v --all-features
+        - cargo doc -v --all-features
+        - cargo test -v --all-features
 
     - name: LlamaDb VM example
 
       language: rust
       rust: nightly
       cache: cargo
+      sudo: false
       before_install:
         - rustup component add rustfmt-preview --toolchain nightly
         - cd vm/examples/llamadb
@@ -66,15 +73,15 @@ matrix:
         - cargo doc -v
         - cargo test -v
 
-    - name: Publisher CLI
+    - name: Fluence CLI
 
       language: rust
       rust: stable
       cache: cargo
 
       before_install:
         - rustup component add rustfmt-preview
-        - cd publisher
+        - cd cli
 
       script:
         - npm install --prefix ../bootstrap

bench/vm/.gitignore

@@ -0,0 +1,2 @@
+# bencher log
+wasm_bencher_log

bench/vm/bencher/BenchTestGenerator.py

@@ -0,0 +1,85 @@
+"""
+Copyright 2018 Fluence Labs Limited
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from os.path import join
+from settings import *
+from subprocess import call
+
+
+class BenchTestGenerator:
+    """Generates tests in provided directory."""
+
+    def __init__(self, test_dir):
+        self.tests_dir = test_dir
+        self.generated_tests_dir = "bench_tests"
+
+    def generate_tests(self, out_dir, test_descriptors):
+        """Generates tests by their descriptors.
+
+        Compiles each test by test_generator_cmd in given test descriptor, moves it to out_dir/generated_tests_dir
+        and finally sets test_full_path in each test descriptor.
+
+        Parameters
+        ----------
+        out_dir : str
+            A directory where the resulted test will be saved.
+        test_descriptors : [TestDescriptor]
+            Descriptors of tests that specifies how exactly test should be compiled.
+
+        Returns
+        -------
+        [TestDescriptor]
+            Test descriptors with test_full_path filled.
+
+        """
+        call("mkdir -p {}".format(join(out_dir,self.generated_tests_dir)), shell=True)
+
+        # collect garbage to clean possible previous unsuccessfully build results that prevent cargo to build new tests
+        self.__collect_garbage(out_dir)
+
+        generated_tests_dir_full_path = join(out_dir, self.generated_tests_dir)
+        test_mv_cmd = "mv " + join(out_dir, "wasm32-unknown-unknown", "release", "{}.wasm") + " " \
+                            + join(generated_tests_dir_full_path, "{}.wasm")
+
+        for test_name, test_descriptor in test_descriptors.items():
+            test_full_path = join(self.tests_dir, test_descriptor.test_folder_name)
+            test_compilation_cmd = test_descriptor.test_compilation_cmd.format(test_full_path, out_dir)
+
+            for key, value in test_descriptor.test_compilation_parameters.items():
+                test_compilation_cmd = "{}={} {}".format(key, value, test_compilation_cmd)
+
+            call(test_compilation_cmd, shell=True)
+            call(test_mv_cmd.format(test_descriptor.test_folder_name, test_name), shell=True)
+
+            # collect garbage to force cargo build the same test with different env params again
+            self.__collect_garbage(out_dir)
+
+            test_descriptors[test_name].generated_test_full_path = \
+                join(generated_tests_dir_full_path, "{}.wasm").format(test_name)
+
+        return test_descriptors
+
+    def __collect_garbage(self, out_dir):
+        """Removes rust cargo target directories.
+
+        Attributes
+        ----------
+        out_dir : str
+            A directory where build results are placed.
+
+        """
+        # TODO : clean by cargo clean
+        call(("rm -rf " + join("{}", "wasm32-unknown-unknown")).format(out_dir), shell=True)
+        call(("rm -rf " + join("{}", "release")).format(out_dir), shell=True)

bench/vm/bencher/TestDescriptor.py

@@ -0,0 +1,37 @@
+"""
+Copyright 2018 Fluence Labs Limited
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+
+class TestDescriptor:
+    """Descriptor of test written in Rust that contains parameters controls test compilation process.
+
+    Attributes
+    ----------
+    folder_name : str
+        A name of folder where test is located.
+    compilation_cmd : str
+        A compilation string that has to be used to build the test.
+    compilation_parameters : {parameter : parameter_value}
+        Parameters that used in test compilation.
+    generated_test_full_path : str
+        A full path of finally compiled test.
+
+    """
+    def __init__(self, test_folder_name="", test_generator_cmd="", test_generator_parameters={}):
+        self.test_folder_name = test_folder_name
+        self.test_compilation_cmd = test_generator_cmd
+        self.test_compilation_parameters = test_generator_parameters
+        self.generated_test_full_path = ""

bench/vm/bencher/VMDescriptor.py

@@ -0,0 +1,34 @@
+"""
+Copyright 2018 Fluence Labs Limited
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+
+class VMDescriptor:
+    """Wasm VM descriptor that specifies how vm has to be launched.
+
+    Attributes
+    ----------
+    relative_vm_binary_path : str
+        A relative path to vm binary in its main folder.
+    vm_launch_cmd : str
+        An format string with command for launch this vm with provided test.
+    is_compiler_type : bool
+        True, if vm is compiler-type (JIT, AOT, ...).
+
+    """
+    def __init__(self, vm_relative_binary_path="", vm_launch_cmd="", is_compiler_type=True):
+        self.vm_relative_binary_path = vm_relative_binary_path
+        self.vm_launch_cmd = vm_launch_cmd
+        self.is_compiler_type = is_compiler_type

bench/vm/bencher/WasmVMBencher.py

@@ -0,0 +1,107 @@
+"""
+Copyright 2018 Fluence Labs Limited
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from settings import interpreter_launch_count, compiler_launch_count, test_function_name
+
+from os import listdir
+from os.path import join
+from time import time
+from subprocess import Popen
+from collections import defaultdict
+import logging
+
+
+class Record:
+    """Contains measures of one test launch.
+
+    Attributes
+    ----------
+    time : time_type
+        The execution time of one test.
+    cpu_load : int
+        The cpu load (in percents) of one test (currently not supported).
+
+    """
+    def __init__(self, time=0, cpu_load=0):
+        self.time = time
+        self.cpu_load = cpu_load  # TODO
+
+
+class WasmVMBencher:
+    """Launches each VM on given directory on each provided test."""
+
+    def __init__(self, vm_dir):
+        self.vm_dir = vm_dir
+        self.enabled_vm = listdir(vm_dir)
+
+    def run_tests(self, test_descriptors, vm_descriptors):
+        """Launches provided tests and returns their execution time.
+
+        Parameters
+        ----------
+        test_descriptors
+            Descriptors of test that should be used for benchmark Wasm VM.
+        vm_descriptors
+            Descriptors of Wasm VM that should be tested on provided tests.
+
+        Returns
+        -------
+        {vm : {test_name : [Records]}}
+            Collected test results.
+
+       """
+        # {vm : {test_name : [Records]}}
+        results = defaultdict(lambda: defaultdict(list))
+        logger = logging.getLogger("wasm_bencher_logger")
+
+        for test_name, test_descriptor in test_descriptors.items():
+            logger.info("<wasm_bencher>: launch {} test".format(test_name))
+            for vm in self.enabled_vm:
+                if vm not in vm_descriptors:
+                    continue
+
+                vm_binary_full_path = join(self.vm_dir, vm, vm_descriptors[vm].vm_relative_binary_path)
+                cmd = vm_binary_full_path + " " \
+                      + vm_descriptors[vm].vm_launch_cmd.format(wasm_file_path=test_descriptor.generated_test_full_path,
+                                                                function_name=test_function_name)
+
+                launch_count = compiler_launch_count if vm_descriptors[vm].is_compiler_type \
+                    else interpreter_launch_count
+                for _ in range(launch_count):
+                    logger.info("<wasm_bencher>: {}".format(cmd))
+                    result_record = self.__do_one_test(cmd)
+                    results[vm][test_name].append(result_record)
+                    logger.info("<wasm_bencher>: {} result collected: time={}".format(vm, result_record.time))
+
+        return results
+
+    def __do_one_test(self, vm_cmd):
+        """Launches provided shell command string via subprocess.Popen and measure its execution time.
+
+        Parameters
+        ----------
+        vm_cmd : str
+            An exactly command that should be executed.
+
+        Returns
+        -------
+        time_type
+            An elapsed time of provided cmd execution.
+
+        """
+        start_time = time()
+        Popen(vm_cmd, shell=True).wait(None)
+        end_time = time()
+        return Record(end_time - start_time)

bench/vm/bencher/main.py

@@ -0,0 +1,70 @@
+#!/usr/bin/python
+
+"""
+Copyright 2018 Fluence Labs Limited
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from BenchTestGenerator import BenchTestGenerator
+from WasmVMBencher import WasmVMBencher
+from settings import vm_descriptors, test_descriptors
+import click
+import csv
+import logging
+from os.path import join
+
+
+def save_test_results(out_dir, results):
+    """Saves provided results to <vm_name>.csv files in given out_dir.
+
+    Parameters
+    ----------
+    out_dir : str
+        A directory where the result will be saved.
+    results : {vm_name : {test_name : [Record]}}
+        Results that should be saved.
+    """
+    for vm in results:
+        with open(join(out_dir, vm + ".csv"), 'w', newline='') as bench_result_file:
+            fieldnames = ['test_name', 'elapsed_time']
+            writer = csv.DictWriter(bench_result_file, fieldnames=fieldnames)
+            writer.writeheader()
+
+            for test_name, result_records in results[vm].items():
+                for record in result_records:
+                    writer.writerow({"test_name" : test_name, "elapsed_time" : record.time})
+
+
+@click.command()
+@click.option("--vm_dir", help="directory with Webassembly virtual machines")
+@click.option("--tests_dir", help="directory with benchmark tests")
+@click.option("--out_dir", help="directory where results will be saved")
+def main(vm_dir, tests_dir, out_dir):
+    logging.basicConfig(filename="wasm_bencher_log", level=logging.INFO, format='%(asctime)s %(message)s',
+                        datefmt='%m/%d/%Y %I:%M:%S %p')
+
+    logger = logging.getLogger("wasm_bench_logger")
+    logger.info("<wasm_bencher>: starting tests generation")
+    test_generator = BenchTestGenerator(tests_dir)
+    filled_tests_descriptors = test_generator.generate_tests(out_dir, test_descriptors)
+
+    logger.info("<wasm_bencher>: starting vm tests")
+    vm_bencher = WasmVMBencher(vm_dir)
+    test_results = vm_bencher.run_tests(filled_tests_descriptors, vm_descriptors)
+
+    logger.info("<wasm_bencher>: starting collection of test results")
+    save_test_results(out_dir, test_results)
+
+
+if __name__ == '__main__':
+    main()