Upgrade model version to 2, remove the binarize layer

common/helper.h

@@ -55,7 +55,7 @@
 #define STR(a) #a
 #define XSTR(a) STR(a)
 
-#define PNT_STR(s) << s << " "
+#define PNT_STR(s) << s
 #define PNT_VAR(var) << XSTR(var) << " = " << (var) << ", "
 #define PNT_TO(stream, ...) stream FOR_EACH(PNT_VAR, __VA_ARGS__);
 #define PNT(...) PNT_TO(LOG(INFO), __VA_ARGS__)

common/macros.h

@@ -0,0 +1,8 @@
+// Copyright 2019 JD.com Inc. JD AI
+
+#ifndef BNN_MACROS_H
+#define BNN_MACROS_H
+
+#define BNN_LATEST_MODEL_VERSION 2
+
+#endif

dabnn/bitpack.h

@@ -25,6 +25,8 @@
 #include <glog/logging.h>
 #include "mat.h"
 
+namespace bnn {
+
 #ifdef __aarch64__
 inline void pack_128_opt(const float *float_ptr, void *binary_ptr,
                          size_t size) {
@@ -275,4 +277,5 @@ inline void pack_mat(const bnn::Mat &float_mat, bnn::Mat &binary_mat) {
 #endif  // __aarch64__
 }
 
+}
 #endif /* BITPACK_H */

dabnn/layers/BinConv.cpp

@@ -5,6 +5,8 @@
 #include <common/baseline.h>
 #include <dabnn/bconv.h>
 #include <dabnn/bgemm.h>
+#include <dabnn/bitpack.h>
+#include <dabnn/fused_binarize_im2col.h>
 #include <dabnn/net.h>
 #include <dabnn/pad.h>
 
@@ -21,7 +23,14 @@ BinConv::BinConv(NetCP net, const std::string &name, css input, css weight,
       stride_h(stride_h),
       stride_w(stride_w) {
     auto &mat_map = net.lock()->mat_map_;
-    const auto &pad_name = "pad_for_" + output + "_cal";
+    const auto binaized_name = "binaized_for_" + output + "_cal";
+    if (mat_map.find(binaized_name) == mat_map.end()) {
+        auto &input_mat = *mat_map[input];
+        mat_map[binaized_name] = std::make_shared<Mat>(
+            input_mat.h, input_mat.w, input_mat.elem_c,
+            DataType::Bit, binaized_name);
+    }
+    const auto pad_name = "pad_for_" + output + "_cal";
     if (mat_map.find(pad_name) == mat_map.end()) {
         auto &input_mat = *mat_map[input];
         mat_map[pad_name] = std::make_shared<Mat>(
@@ -45,8 +54,8 @@ BinConv::BinConv(NetCP net, const std::string &name, css input, css weight,
         const int m = weight_mat->n;
         BNN_ASSERT(weight_mat->total() % m == 0, "");
         const int k = weight_mat->total() / m;
-        transposed_weight_mat = std::make_shared<Mat>(
-            m, k * 64, DataType::Bit, false);
+        transposed_weight_mat =
+            std::make_shared<Mat>(m, k * 64, DataType::Bit, false);
         auto *trans_data_ptr =
             static_cast<uint64_t *>(transposed_weight_mat->data);
         auto *data_ptr = static_cast<uint64_t *>(weight_mat->data);
@@ -96,25 +105,27 @@ bool BinConv::gemm_compatible() const {
 void BinConv::forward_impl() const {
     if (net_.lock()->optimize) {
         if (direct_conv_compatible()) {
-            pad(*input_mat, pad_h, pad_w, *padded_mat);
+            pack_mat(*input_mat, *binarized_mat);
+            pad(*binarized_mat, pad_h, pad_w, *padded_mat);
             bconv_3x3(*padded_mat, *weight_mat, *output_mat, stride_h);
         } else if (gemm_compatible()) {
             output_mat->fill<float>(0.f);
-            bnn::im2col(*input_mat, weight_mat->h, weight_mat->w, pad_h, pad_w,
-                        stride_h, stride_w, 1, 1, *col_mat);
+            bnn::fused_binarize_im2col(*input_mat, weight_mat->h, weight_mat->w, pad_h, pad_w, stride_h, stride_w, 1, 1, *col_mat);
             const int m = weight_mat->n;
             const int n = output_mat->h * output_mat->w;
             const int k = weight_mat->h * weight_mat->w * weight_mat->c;
             bgemm(m, n, k, static_cast<uint64_t *>(transposed_weight_mat->data),
                   m, static_cast<uint64_t *>(col_mat->data), k,
                   static_cast<float *>(output_mat->data), m);
         } else {
-            baseline_bconv(*input_mat, *weight_mat, weight_mat->h,
+            pack_mat(*input_mat, *binarized_mat);
+            baseline_bconv(*binarized_mat, *weight_mat, weight_mat->h,
                            weight_mat->w, pad_h, pad_w, stride_h, stride_w, 1,
                            1, output_mat->c, *output_mat);
         }
     } else {
-        baseline_bconv(*input_mat, *weight_mat, weight_mat->h, weight_mat->w,
+        pack_mat(*input_mat, *binarized_mat);
+        baseline_bconv(*binarized_mat, *weight_mat, weight_mat->h, weight_mat->w,
                        pad_h, pad_w, stride_h, stride_w, 1, 1, output_mat->c,
                        *output_mat);
     }

dabnn/layers/BinConv.h

@@ -9,6 +9,7 @@ namespace bnn {
 class BinConv : public Layer {
    public:
     MatCP input_mat;
+    MatP binarized_mat;
     MatP padded_mat;
     MatP col_mat;
     MatCP weight_mat;

dabnn/net.cpp

@@ -9,6 +9,7 @@
 #include <vector>
 
 #include <common/flatbuffers_helper.h>
+#include <common/macros.h>
 #include <dabnn/bitpack.h>
 #include <dabnn/layers/Add.h>
 #include <dabnn/layers/Affine.h>
@@ -54,6 +55,9 @@ void Net::read_impl(const void *ptr) {
 
 void Net::prepare() {
     BNN_ASSERT(!(strict && !run_fconv), "fconv must be run in strict mode");
+    BNN_ASSERT(model_->version() == BNN_LATEST_MODEL_VERSION,
+               "The model version should be ", BNN_LATEST_MODEL_VERSION,
+               ", got ", model_->version(), " instead.");
     for (const auto &tensor : *model_->inputs()) {
         Shaper::Shape shape(tensor->shape()->begin(), tensor->shape()->end());
         const auto name = tensor->name()->str();
@@ -94,9 +98,9 @@ void Net::prepare() {
                     FORZ(j, 64) { float_data[i * 64 + j] = bs[j] ? 1 : -1; }
                 }
 
-                add_mat(name, std::make_shared<Mat>(shape[0], shape[1],
-                                                    shape[2], shape[3],
-                                                    bnn::DataType::Bit, len, false));
+                add_mat(name, std::make_shared<Mat>(
+                                  shape[0], shape[1], shape[2], shape[3],
+                                  bnn::DataType::Bit, len, false));
                 pack_mat_128(*tmp, *mat_map_[name]);
             } else {
 #endif  // __aarch64__
@@ -174,9 +178,8 @@ void Net::prepare() {
                 break;
             }
             case flatbnn::LayerType::BinConv2D: {
-                ADD_LAYER_WITH_DATA_TYPE(bin_conv2d, Conv, DataType::Float,
-                                         input, strides, dilations, pads,
-                                         weight, output);
+                ADD_LAYER(bin_conv2d, Conv, input, strides, dilations, pads,
+                          weight, output);
                 BNN_ASSERT(pads.size() == 2 ||
                                (pads.size() == 4 && pads[0] == pads[2] &&
                                 pads[1] == pads[3]),

tools/onnx2bnn/OnnxConverter.cpp

@@ -15,6 +15,7 @@
 #include <common/common_bitpack.h>
 #include <common/flatbuffers_helper.h>
 #include <common/helper.h>
+#include <common/macros.h>
 #include <glog/logging.h>
 #include <onnx/onnx_pb.h>
 #include <onnx/optimizer/optimize.h>
@@ -44,15 +45,6 @@ void OnnxConverter::AddBinConv(const std::string &input_name,
                                BTensor bin_weight) {
     css bin_name = input_name + "_bin";
 
-    {
-        const auto param = flatbnn::CreateBinarizeDirect(
-            builder_, input_name.c_str(), bin_name.c_str());
-        const auto layer =
-            flatbnn::CreateLayer(builder_, flatbnn::LayerType::Binarize, 0, 0,
-                                 0, 0, 0, 0, 0, 0, 0, 0, param);
-        layers_.push_back(layer);
-    }
-
     BNN_ASSERT(group == 1, "Group != 1 is not supported");
     const auto param = flatbnn::CreateBinConv2DDirect(
         builder_, bin_name.c_str(), weight_name.c_str(), nullptr, &pads,
@@ -507,7 +499,8 @@ std::vector<std::string> OnnxConverter::Convert(
     auto flat_inputs = builder_.CreateVector(inputs);
     auto flat_tensors = builder_.CreateVector(tensors_);
     auto flat_model =
-        flatbnn::CreateModel(builder_, flat_layers, flat_tensors, flat_inputs);
+        flatbnn::CreateModel(builder_, flat_layers, flat_tensors, flat_inputs,
+                             BNN_LATEST_MODEL_VERSION);
 
     builder_.Finish(flat_model);