[Runtime] MISRA-C compliant TVM runtime

apps/bundle_deploy_c/Makefile

@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Makefile Example to bundle TVM modules.
+
+# Setup build environment
+TVM_ROOT=$(shell cd ../..; pwd)
+DMLC_CORE=${TVM_ROOT}/3rdparty/dmlc-core
+PKG_CXXFLAGS = -std=c++14 -O2 -fPIC \
+	-I${TVM_ROOT}/include \
+	-I${DMLC_CORE}/include \
+	-I${TVM_ROOT}/3rdparty/dlpack/include
+PKG_CFLAGS = -std=c99 -O2 -fPIC \
+	-I${TVM_ROOT}/include \
+	-I${DMLC_CORE}/include \
+	-I${TVM_ROOT}/3rdparty/dlpack/include
+
+PKG_LDFLAGS = -pthread
+
+build_dir := build
+
+test: $(build_dir)/demo $(build_dir)/bundle.so $(build_dir)/cat.bin
+	$(build_dir)/demo $(build_dir)/bundle.so $(build_dir)/cat.bin
+
+$(build_dir)/demo: demo.cc ${build_dir}/graph.json.c ${build_dir}/params.bin.c
+	@mkdir -p $(@D)
+	g++ $(PKG_CXXFLAGS) -o $@  demo.cc -ldl
+
+# Serialize our graph.json file.
+$(build_dir)/graph.json.c: $(build_dir)/graph.json
+	xxd -i $^  > $@
+
+# Serialize our params.bin file.
+$(build_dir)/params.bin.c: $(build_dir)/params.bin
+	xxd -i $^  > $@
+
+$(build_dir)/model.o $(build_dir)/graph.json $(build_dir)/params.bin $(build_dir)/cat.bin: build_model.py
+	python3 $< -o $(build_dir)
+
+# Build our bundle against the serialized bundle.c API, the runtime.cc API, and
+# the serialized graph.json and params.bin
+$(build_dir)/bundle.so: bundle.c runtime.c $(build_dir)/model.o
+	@mkdir -p $(@D)
+	gcc -shared $(PKG_CFLAGS) -fvisibility=hidden -o $@  $^ $(PKG_LDFLAGS)
+
+clean:
+	rm -rf $(build_dir)/bundle.so
+
+cleanall:
+	rm -rf $(build_dir)

apps/bundle_deploy_c/README.md

@@ -0,0 +1,53 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+
+How to Bundle TVM Modules
+=========================
+
+This folder contains an example on how to bundle a TVM module (with the required
+interpreter runtime modules such as `runtime::GraphRuntime`, the graph JSON, and
+the params) into a single, self-contained shared object (`bundle.so`) which
+exposes a C API wrapping the appropriate `runtime::GraphRuntime` instance.
+
+This is useful for cases where we'd like to avoid deploying the TVM runtime
+components to the target host in advance - instead, we simply deploy the bundled
+shared-object to the host, which embeds both the model and the runtime
+components. The bundle should only depend on libc/libc++.
+
+It also contains an example code (`demo.cc`) to load this shared object and
+invoke the packaged TVM model instance. This is a dependency-free binary that
+uses the functionality packaged in `bundle.so` (which means that `bundle.so` can
+be deployed lazily at runtime, instead of at compile time) to invoke TVM
+functionality.
+
+Type the following command to run the sample code under the current folder,
+after building TVM first.
+
+```bash
+make demo
+```
+
+This will:
+
+- Download the mobilenet0.25 model from the MXNet Gluon Model Zoo
+- Compile the model with NNVM
+- Build a `bundle.so` shared object containing the model specification and
+  parameters
+- Build a `demo` executable that `dlopen`'s `bundle.so`, instantiates the
+  contained graph runtime, and invokes the `GraphRuntime::Run` function on a
+  random input, then prints the output tensor to `stderr`.

apps/bundle_deploy_c/build_model.py

@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Creates a simple TVM modules."""
+
+import argparse
+import os
+from tvm import relay
+import tvm
+import logging
+import json
+
+def build_module(opts):
+    dshape = (1, 3, 224, 224)
+    from mxnet.gluon.model_zoo.vision import get_model
+    block = get_model('mobilenet0.25', pretrained=True)
+    shape_dict = {'data': dshape}
+    mod, params = relay.frontend.from_mxnet(block, shape_dict)
+    func = mod["main"]
+    func = relay.Function(func.params, relay.nn.softmax(func.body), None, func.type_params, func.attrs)
+
+    with relay.build_config(opt_level=3):
+        graph, lib, params = relay.build(
+            func, 'llvm --system-lib', params=params)
+
+    build_dir = os.path.abspath(opts.out_dir)
+    if not os.path.isdir(build_dir):
+        os.makedirs(build_dir)
+
+    lib.save(os.path.join(build_dir, 'model.o'))
+    with open(os.path.join(build_dir, 'graph.json'), 'w') as f_graph_json:
+        f_graph_json.write(graph)
+    with open(os.path.join(build_dir, 'params.bin'), 'wb') as f_params:
+        f_params.write(relay.save_param_dict(params))
+
+def build_inputs(opts):
+    from tvm.contrib import download
+    from PIL import Image
+    import numpy as np
+
+    build_dir = os.path.abspath(opts.out_dir)
+
+    # Download test image
+    image_url = 'https://homes.cs.washington.edu/~moreau/media/vta/cat.jpg'
+    image_fn = os.path.join(build_dir, "cat.png")
+    download.download(image_url, image_fn)
+    image = Image.open(image_fn).resize((224, 224))
+
+    def transform_image(image):
+        image = np.array(image) - np.array([123., 117., 104.])
+        image /= np.array([58.395, 57.12, 57.375])
+        image = image.transpose((2, 0, 1))
+        image = image[np.newaxis, :]
+        return image
+
+    x = transform_image(image)
+    print('x', x.shape)
+    with open(os.path.join(build_dir, "cat.bin"), "wb") as fp:
+        fp.write(x.astype(np.float32).tobytes())
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.INFO)
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-o', '--out-dir', default='.')
+    opts = parser.parse_args()
+
+    build_module(opts)
+    build_inputs(opts)

apps/bundle_deploy_c/bundle.c

@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+/*! \brief macro to do C API call */
+#define TVM_CCALL(func)                                                 \
+  do {                                                                  \
+    int ret = (func);                                                   \
+    if (ret != 0) {                                                     \
+      fprintf(stderr, "%s: %d: error: %s\n", __FILE__, __LINE__, TVMGetLastError()); \
+      exit(ret);                                                        \
+    }                                                                   \
+  } while (0)
+
+TVM_DLL void * tvm_runtime_create(const char * json_data,
+                                  const char * params_data,
+                                  const uint64_t params_size) {
+  int64_t device_type = kDLCPU;
+  int64_t device_id = 0;
+
+  TVMByteArray params;
+  params.data = params_data;
+  params.size = params_size;
+
+  TVMContext ctx;
+  ctx.device_type = (DLDeviceType)device_type;
+  ctx.device_id = device_id;
+
+  // declare pointers
+  TVMModuleHandle (*SystemLibraryCreate)();
+  TVMModuleHandle (*TVMGraphRuntimeCreate)(const char *, const TVMModuleHandle, const TVMContext *);
+  int (*TVMGraphRuntime_LoadParams)(TVMModuleHandle, const char *, const uint32_t);
+
+  // get pointers
+  TVM_CCALL(TVMFuncGetGlobal("runtime.SystemLib", (TVMFunctionHandle*)&SystemLibraryCreate));
+  TVM_CCALL(TVMFuncGetGlobal("tvm.graph_runtime.create", (TVMFunctionHandle*)&TVMGraphRuntimeCreate));
+
+  // run modules
+  TVMModuleHandle mod_syslib = SystemLibraryCreate();
+  TVMModuleHandle mod = TVMGraphRuntimeCreate(json_data, mod_syslib, &ctx);
+  TVM_CCALL(TVMModGetFunction(mod, "load_params", 0, (TVMFunctionHandle*)&TVMGraphRuntime_LoadParams));
+  TVMGraphRuntime_LoadParams(mod, params.data, params.size);
+
+  return mod;
+}
+
+TVM_DLL void tvm_runtime_destroy(void * runtime) {
+  void (*TVMGraphRuntimeRelease)(TVMModuleHandle *);
+  TVM_CCALL(TVMFuncGetGlobal("tvm.graph_runtime.release", (TVMFunctionHandle*)&TVMGraphRuntimeRelease));
+  TVMGraphRuntimeRelease(&runtime);
+}
+
+TVM_DLL void tvm_runtime_set_input(void * runtime, const char * name, DLTensor * tensor) {
+  void (*TVMGraphRuntime_SetInput)(TVMModuleHandle, const char *, DLTensor*);
+  TVM_CCALL(TVMFuncGetGlobal("tvm.graph_runtime.set_input", (TVMFunctionHandle*)&TVMGraphRuntime_SetInput));
+  TVMGraphRuntime_SetInput(runtime, "data", tensor);
+}
+
+TVM_DLL void tvm_runtime_run(void * runtime) {
+  void (*TVMGraphRuntime_Run)(TVMModuleHandle runtime);
+  TVM_CCALL(TVMFuncGetGlobal("tvm.graph_runtime.run", (TVMFunctionHandle*)&TVMGraphRuntime_Run));
+  TVMGraphRuntime_Run(runtime);
+}
+
+TVM_DLL void tvm_runtime_get_output(void * runtime, int32_t index, DLTensor * tensor) {
+  int (*TVMGraphRuntime_GetOutput)(TVMModuleHandle, const int32_t, DLTensor *);
+  TVM_CCALL(TVMFuncGetGlobal("tvm.graph_runtime.get_output", (TVMFunctionHandle*)&TVMGraphRuntime_GetOutput));
+  TVMGraphRuntime_GetOutput(runtime, index, tensor);
+}
+

apps/bundle_deploy_c/demo.cc

@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+
+#include <assert.h>
+#include <dlfcn.h> //dlopen
+#include <iostream>
+#include <random>
+#include <vector>
+
+#include "build/graph.json.c"
+#include "build/params.bin.c"
+
+template <typename F> auto getFunc(void *bundle, const char *name) {
+  dlerror();
+  auto *f =
+      reinterpret_cast<typename std::add_pointer<F>::type>(dlsym(bundle, name));
+  assert(!dlerror());
+  return f;
+}
+
+int main(int argc, char **argv) {
+  assert(argc == 3 && "Usage: demo <bundle.so> <cat.bin>");
+  auto *bundle = dlopen(argv[1], RTLD_LAZY | RTLD_LOCAL);
+  assert(bundle);
+
+  char * json_data = reinterpret_cast<char*>(build_graph_json);
+  char * params_data = reinterpret_cast<char*>(build_params_bin);
+  uint64_t params_size = build_params_bin_len;
+
+  auto *handle = getFunc<void *(char*, char*, int)>(bundle, "tvm_runtime_create")(
+      json_data, params_data, params_size);
+
+  float input_storage[1 * 3 * 224 * 224];
+  FILE * fp = fopen(argv[2], "rb");
+  fread(input_storage, 3 * 224 * 224, 4, fp);
+  fclose(fp);
+
+  std::vector<int64_t> input_shape = {1, 3, 224, 224};
+  DLTensor input;
+  input.data = input_storage;
+  input.ctx = DLContext{kDLCPU, 0};
+  input.ndim = 4;
+  input.dtype = DLDataType{kDLFloat, 32, 1};
+  input.shape = input_shape.data();
+  input.strides = nullptr;
+  input.byte_offset = 0;
+
+  getFunc<void(void *, const char *, void *)>(bundle, "tvm_runtime_set_input")(
+      handle, "data", &input);
+
+  auto *ftvm_runtime_run =
+      (auto (*)(void *)->void)dlsym(bundle, "tvm_runtime_run");
+  assert(!dlerror());
+  ftvm_runtime_run(handle);
+
+  float output_storage[1000];
+  std::vector<int64_t> output_shape = {1, 1000};
+  DLTensor output;
+  output.data = output_storage;
+  output.ctx = DLContext{kDLCPU, 0};
+  output.ndim = 2;
+  output.dtype = DLDataType{kDLFloat, 32, 1};
+  output.shape = output_shape.data();
+  output.strides = nullptr;
+  output.byte_offset = 0;
+
+  getFunc<void(void *, int, void *)>(bundle, "tvm_runtime_get_output")(
+      handle, 0, &output);
+  float max_iter = -std::numeric_limits<float>::max();
+  int32_t max_index = -1;
+  for (auto i = 0; i < 1000; ++i) {
+    if (output_storage[i] > max_iter) {
+      max_iter = output_storage[i];
+      max_index = i;
+    }
+  }
+  printf("The maximum position in output vector is: %d, with max-value %f.\n",
+         max_index, max_iter);
+  getFunc<void(void *)>(bundle, "tvm_runtime_destroy")(handle);
+  dlclose(bundle);
+  return 0;
+}