Merge pull request envoyproxy#482 from kyessenov/flatbuffer_impl

implement declare_property

bazel/repositories.bzl

@@ -152,6 +152,7 @@ def envoy_dependencies(skip_targets = []):
     _proxy_wasm_cpp_host()
     _repository_impl("com_googlesource_code_re2")
     _com_google_cel_cpp()
+    _repository_impl("com_github_google_flatbuffers")
     _repository_impl("bazel_toolchains")
     _repository_impl("bazel_compdb")
     _repository_impl("envoy_build_tools")

bazel/repository_locations.bzl

@@ -296,10 +296,15 @@ REPOSITORY_LOCATIONS = dict(
         urls = ["https://storage.googleapis.com/quiche-envoy-integration/googleurl_dbf5ad147f60afc125e99db7549402af49a5eae8.tar.gz"],
     ),
     com_google_cel_cpp = dict(
-        sha256 = "d942a8d2e5831bcf7f5b1e99b07f90534eb082f40fd9bda05bcc24ff9c0c3571",
-        strip_prefix = "cel-cpp-a9eec4686b72c28980a09fe2e253ec897a781c32",
-        # 2019-12-09
-        urls = ["https://github.com/google/cel-cpp/archive/a9eec4686b72c28980a09fe2e253ec897a781c32.tar.gz"],
+        sha256 = "7eddffdb231e7c82f60c597cd38e742fbc0c48f54ca33015ac0a3d22bd51bba3",
+        strip_prefix = "cel-cpp-d88a4822af1864b481b31b12c2ecc4e631a7f8a9",
+        # 2019-04-13
+        urls = ["https://github.com/google/cel-cpp/archive/d88a4822af1864b481b31b12c2ecc4e631a7f8a9.tar.gz"],
+    ),
+    com_github_google_flatbuffers = dict(
+        sha256 = "b8efbc25721e76780752bad775a97c3f77a0250271e2db37fc747b20e8b0f24a",
+        strip_prefix = "flatbuffers-a83caf5910644ba1c421c002ef68e42f21c15f9f",
+        urls = ["https://github.com/google/flatbuffers/archive/a83caf5910644ba1c421c002ef68e42f21c15f9f.tar.gz"],
     ),
     com_googlesource_code_re2 = dict(
         sha256 = "04ee2aaebaa5038554683329afc494e684c30f82f2a1e47eb62450e59338f84d",

source/extensions/common/wasm/BUILD

@@ -4,6 +4,7 @@ load(
     "//bazel:envoy_build_system.bzl",
     "envoy_cc_library",
     "envoy_package",
+    "envoy_proto_library",
 )
 
 envoy_package()
@@ -23,6 +24,7 @@ envoy_cc_library(
         "context.h",
         "foreign.h",
         "wasm.h",
+        "wasm_state.h",
         "wasm_vm.h",
     ],
     deps = [
@@ -32,6 +34,7 @@ envoy_cc_library(
         "//include/envoy/server:wasm_interface",
         "//include/envoy/upstream:cluster_manager_interface",
         "//source/common/config:datasource_lib",
+        "//source/common/singleton:const_singleton",
         "//source/common/stats:stats_lib",
         "//source/extensions/filters/common/expr:evaluator_lib",
         "//source/extensions/filters/http:well_known_names",
@@ -53,6 +56,10 @@ envoy_cc_library(
     deps = [
         "//include/envoy/stream_info:filter_state_interface",
         "//source/common/protobuf",
+        "//source/common/singleton:const_singleton",
+        "@com_github_google_flatbuffers//:flatbuffers",
+        "@com_google_cel_cpp//eval/public:cel_value",
+        "@com_google_cel_cpp//tools:flatbuffers_backed_impl",
     ],
 )
 
@@ -65,6 +72,7 @@ envoy_cc_library(
         "wasm_vm.cc",
     ],
     deps = [
+        ":declare_property_cc_proto",
         ":wasm_hdr",
         ":wasm_interoperation_lib",
         "//external:abseil_base",
@@ -88,3 +96,8 @@ envoy_cc_library(
         "@proxy_wasm_cpp_host//:lib",
     ],
 )
+
+envoy_proto_library(
+    name = "declare_property",
+    srcs = ["declare_property.proto"],
+)

source/extensions/common/wasm/context.cc

@@ -23,7 +23,6 @@
 #include "common/tracing/http_tracer_impl.h"
 
 #include "extensions/common/wasm/wasm.h"
-#include "extensions/common/wasm/wasm_state.h"
 #include "extensions/common/wasm/well_known_names.h"
 #include "extensions/filters/common/expr/context.h"
 
@@ -409,47 +408,9 @@ WasmResult serializeValue(Filters::Common::Expr::CelValue value, std::string* re
   return WasmResult::SerializationFailure;
 }
 
-// An expression wrapper for the WASM state
-class WasmStateWrapper : public google::api::expr::runtime::CelMap {
-public:
-  WasmStateWrapper(const StreamInfo::FilterState& filter_state,
-                   const StreamInfo::FilterState* upstream_connection_filter_state)
-      : filter_state_(filter_state),
-        upstream_connection_filter_state_(upstream_connection_filter_state) {}
-  absl::optional<google::api::expr::runtime::CelValue>
-  operator[](google::api::expr::runtime::CelValue key) const override {
-    if (!key.IsString()) {
-      return {};
-    }
-    auto value = key.StringOrDie().value();
-    if (filter_state_.hasData<WasmState>(value)) {
-      const WasmState& result = filter_state_.getDataReadOnly<WasmState>(value);
-      return google::api::expr::runtime::CelValue::CreateBytes(&result.value());
-    }
-
-    if (upstream_connection_filter_state_ &&
-        upstream_connection_filter_state_->hasData<WasmState>(value)) {
-      const WasmState& result =
-          upstream_connection_filter_state_->getDataReadOnly<WasmState>(value);
-      return google::api::expr::runtime::CelValue::CreateBytes(&result.value());
-    }
-    return {};
-  }
-
-  int size() const override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; }
-  bool empty() const override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; }
-  const google::api::expr::runtime::CelList* ListKeys() const override {
-    NOT_IMPLEMENTED_GCOVR_EXCL_LINE;
-  }
-
-private:
-  const StreamInfo::FilterState& filter_state_;
-  const StreamInfo::FilterState* upstream_connection_filter_state_;
-};
-
 #define PROPERTY_TOKENS(_f)                                                                        \
-  _f(METADATA) _f(FILTER_STATE) _f(REQUEST) _f(RESPONSE) _f(CONNECTION) _f(UPSTREAM) _f(NODE)      \
-      _f(SOURCE) _f(DESTINATION) _f(LISTENER_DIRECTION) _f(LISTENER_METADATA) _f(CLUSTER_NAME)     \
+  _f(METADATA) _f(REQUEST) _f(RESPONSE) _f(CONNECTION) _f(UPSTREAM) _f(NODE) _f(SOURCE)            \
+      _f(DESTINATION) _f(LISTENER_DIRECTION) _f(LISTENER_METADATA) _f(CLUSTER_NAME)                \
           _f(CLUSTER_METADATA) _f(ROUTE_NAME) _f(ROUTE_METADATA) _f(PLUGIN_NAME)                   \
               _f(PLUGIN_ROOT_ID) _f(PLUGIN_VM_ID)
 
@@ -470,25 +431,33 @@ absl::optional<google::api::expr::runtime::CelValue>
 Context::FindValue(absl::string_view name, Protobuf::Arena* arena) const {
   using google::api::expr::runtime::CelValue;
 
+  const StreamInfo::StreamInfo* info = getConstRequestStreamInfo();
+
   // Convert into a dense token to enable a jump table implementation.
   auto part_token = property_tokens.find(name);
   if (part_token == property_tokens.end()) {
+    if (info) {
+      std::string key;
+      absl::StrAppend(&key, WasmStateKeyPrefix, name);
+      const WasmState* state;
+      if (info->filterState().hasData<WasmState>(key)) {
+        state = &info->filterState().getDataReadOnly<WasmState>(key);
+      } else if (info->upstreamFilterState()->hasData<WasmState>(key)) {
+        state = &info->upstreamFilterState()->getDataReadOnly<WasmState>(key);
+      } else {
+        return {};
+      }
+      return state->exprValue(arena);
+    }
     return {};
   }
 
-  const StreamInfo::StreamInfo* info = getConstRequestStreamInfo();
   switch (part_token->second) {
   case PropertyToken::METADATA:
     if (info) {
       return CelValue::CreateMessage(&info->dynamicMetadata(), arena);
     }
     break;
-  case PropertyToken::FILTER_STATE:
-    if (info) {
-      return CelValue::CreateMap(Protobuf::Arena::Create<WasmStateWrapper>(
-          arena, info->filterState(), info->upstreamFilterState().get()));
-    }
-    break;
   case PropertyToken::REQUEST:
     if (info) {
       return CelValue::CreateMap(Protobuf::Arena::Create<Filters::Common::Expr::RequestWrapper>(
@@ -1134,13 +1103,34 @@ const Network::Connection* Context::getConnection() const {
   return nullptr;
 }
 
-WasmResult Context::setProperty(absl::string_view key, absl::string_view serialized_value) {
+WasmResult Context::setProperty(absl::string_view path, absl::string_view value) {
   auto* stream_info = getRequestStreamInfo();
   if (!stream_info) {
     return WasmResult::NotFound;
   }
-  stream_info->filterState()->setData(key, std::make_unique<WasmState>(serialized_value),
-                                      StreamInfo::FilterState::StateType::Mutable);
+  std::string key;
+  absl::StrAppend(&key, WasmStateKeyPrefix, path);
+  WasmState* state;
+  if (stream_info->filterState()->hasData<WasmState>(key)) {
+    state = &stream_info->filterState()->getDataMutable<WasmState>(key);
+  } else {
+    const auto& it = rootContext()->state_prototypes_.find(path);
+    auto state_ptr = std::make_unique<WasmState>(it == rootContext()->state_prototypes_.end()
+                                                     ? DefaultWasmStatePrototype::get()
+                                                     : *it->second.get());
+    state = state_ptr.get();
+    stream_info->filterState()->setData(key, std::move(state_ptr),
+                                        StreamInfo::FilterState::StateType::ReadOnly);
+  }
+  if (!state->setValue(value)) {
+    return WasmResult::BadArgument;
+  }
+  return WasmResult::Ok;
+}
+
+WasmResult Context::declareProperty(absl::string_view path,
+                                    std::unique_ptr<const WasmStatePrototype> state_prototype) {
+  state_prototypes_[path] = std::move(state_prototype);
   return WasmResult::Ok;
 }
 

source/extensions/common/wasm/context.h

@@ -13,6 +13,7 @@
 #include "common/common/assert.h"
 #include "common/common/logger.h"
 
+#include "extensions/common/wasm/wasm_state.h"
 #include "extensions/filters/common/expr/evaluator.h"
 
 #include "eval/public/activation.h"
@@ -192,7 +193,9 @@ class Context : public proxy_wasm::ContextBase,
 
   // State accessors
   WasmResult getProperty(absl::string_view path, std::string* result) override;
-  WasmResult setProperty(absl::string_view key, absl::string_view serialized_value) override;
+  WasmResult setProperty(absl::string_view path, absl::string_view value) override;
+  WasmResult declareProperty(absl::string_view path,
+                             std::unique_ptr<const WasmStatePrototype> state_prototype);
 
   // Continue
   void continueRequest() override;
@@ -266,6 +269,14 @@ class Context : public proxy_wasm::ContextBase,
   absl::optional<google::api::expr::runtime::CelValue>
   FindValue(absl::string_view name, Protobuf::Arena* arena) const override;
   bool IsPathUnknown(absl::string_view) const override { return false; }
+  const std::vector<google::api::expr::runtime::CelAttributePattern>&
+  unknown_attribute_patterns() const override {
+    static const std::vector<google::api::expr::runtime::CelAttributePattern> empty;
+    return empty;
+  }
+  const Protobuf::FieldMask unknown_paths() const override {
+    return Protobuf::FieldMask::default_instance();
+  }
 
   // Foreign function state
   virtual void setForeignData(absl::string_view data_name, std::unique_ptr<StorageObject> data) {
@@ -441,6 +452,9 @@ class Context : public proxy_wasm::ContextBase,
   bool upstream_closed_ = false;
   bool downstream_closed_ = false;
   bool tcp_connection_closed_ = false;
+
+  // Filter state prototype declaration.
+  absl::flat_hash_map<std::string, std::unique_ptr<const WasmStatePrototype>> state_prototypes_;
 };
 using ContextSharedPtr = std::shared_ptr<Context>;
 

source/extensions/common/wasm/declare_property.proto

@@ -0,0 +1,24 @@
+syntax = "proto3";
+
+package envoy.source.extensions.common.wasm;
+
+enum WasmType {
+  Bytes = 0;
+  String = 1;
+  FlatBuffers = 2;
+  Protobuf = 3;
+};
+
+enum LifeSpan {
+  FilterChain = 0;
+  DownstreamRequest = 1;
+  DownstreamConnection = 2;
+};
+
+message DeclarePropertyArguments {
+  string name = 1;
+  bool readonly = 2;
+  WasmType type = 3;
+  bytes schema = 4;
+  LifeSpan span = 5;
+};

source/extensions/common/wasm/foreign.cc

@@ -2,6 +2,8 @@
 
 #include "common/common/logger.h"
 
+#include "source/extensions/common/wasm/declare_property.pb.h"
+
 #include "eval/public/builtin_func_registrar.h"
 #include "eval/public/cel_expr_builder_factory.h"
 #include "parser/parser.h"
@@ -124,7 +126,7 @@ class CreateExpressionFactory : public ExpressionFactory {
       auto token = expr_context.createToken();
       auto& handler = expr_context.getExpression(token);
 
-      handler.parsed_expr_ = parse_status.ValueOrDie();
+      handler.parsed_expr_ = parse_status.value();
       auto cel_expression_status = expr_context.builder()->CreateExpression(
           &handler.parsed_expr_.expr(), &handler.parsed_expr_.source_info());
       if (!cel_expression_status.ok()) {
@@ -133,7 +135,7 @@ class CreateExpressionFactory : public ExpressionFactory {
         return WasmResult::BadArgument;
       }
 
-      handler.compiled_expr_ = std::move(cel_expression_status.ValueOrDie());
+      handler.compiled_expr_ = std::move(cel_expression_status.value());
       auto result = reinterpret_cast<uint32_t*>(alloc_result(sizeof(uint32_t)));
       *result = token;
       return WasmResult::Ok;
@@ -168,7 +170,7 @@ class EvaluateExpressionFactory : public ExpressionFactory {
         return WasmResult::InternalFailure;
       }
       std::string result;
-      auto serialize_status = serializeValue(eval_status.ValueOrDie(), &result);
+      auto serialize_status = serializeValue(eval_status.value(), &result);
       if (serialize_status != WasmResult::Ok) {
         return serialize_status;
       }
@@ -203,6 +205,62 @@ RegisterForeignFunction
     registerDeleteExpressionForeignFunction("expr_delete",
                                             createFromClass<DeleteExpressionFactory>());
 
+// TODO(kyessenov) The factories should be separated into individual compilation units.
+// TODO(kyessenov) Leverage the host argument marshaller instead of the protobuf argument list.
+class DeclarePropertyFactory {
+public:
+  WasmForeignFunction create(std::shared_ptr<DeclarePropertyFactory> self) const {
+    WasmForeignFunction f = [self](WasmBase&, absl::string_view arguments,
+                                   const std::function<void*(size_t size)>&) -> WasmResult {
+      envoy::source::extensions::common::wasm::DeclarePropertyArguments args;
+      if (args.ParseFromArray(arguments.data(), arguments.size())) {
+        WasmType type = WasmType::Bytes;
+        switch (args.type()) {
+        case envoy::source::extensions::common::wasm::WasmType::Bytes:
+          type = WasmType::Bytes;
+          break;
+        case envoy::source::extensions::common::wasm::WasmType::Protobuf:
+          type = WasmType::Protobuf;
+          break;
+        case envoy::source::extensions::common::wasm::WasmType::String:
+          type = WasmType::String;
+          break;
+        case envoy::source::extensions::common::wasm::WasmType::FlatBuffers:
+          type = WasmType::FlatBuffers;
+          break;
+        default:
+          // do nothing
+          break;
+        }
+        StreamInfo::FilterState::LifeSpan span = StreamInfo::FilterState::LifeSpan::FilterChain;
+        switch (args.span()) {
+        case envoy::source::extensions::common::wasm::LifeSpan::FilterChain:
+          span = StreamInfo::FilterState::LifeSpan::FilterChain;
+          break;
+        case envoy::source::extensions::common::wasm::LifeSpan::DownstreamRequest:
+          span = StreamInfo::FilterState::LifeSpan::DownstreamRequest;
+          break;
+        case envoy::source::extensions::common::wasm::LifeSpan::DownstreamConnection:
+          span = StreamInfo::FilterState::LifeSpan::DownstreamConnection;
+          break;
+        default:
+          // do nothing
+          break;
+        }
+        auto context = static_cast<Context*>(proxy_wasm::current_context_);
+        return context->declareProperty(
+            args.name(),
+            std::make_unique<const WasmStatePrototype>(args.readonly(), type, args.schema(), span));
+      }
+      return WasmResult::BadArgument;
+    };
+    return f;
+  }
+};
+RegisterForeignFunction
+    registerDeclarePropertyForeignFunction("declare_property",
+                                           createFromClass<DeclarePropertyFactory>());
+
 } // namespace Wasm
 } // namespace Common
 } // namespace Extensions