Add BigQuery storage (#546)

feast-dev · Apr 3, 2020 · eaeb5df · eaeb5df
1 parent 478f5ce
commit eaeb5df
Show file tree

Hide file tree

Showing 15 changed files with 1,626 additions and 0 deletions.
diff --git a/storage/connectors/bigquery/pom.xml b/storage/connectors/bigquery/pom.xml
@@ -0,0 +1,79 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+    <parent>
+        <groupId>dev.feast</groupId>
+        <artifactId>feast-storage-connectors</artifactId>
+        <version>${revision}</version>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>feast-storage-connector-bigquery</artifactId>
+
+    <name>Feast Storage Connector for BigQuery</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>io.pebbletemplates</groupId>
+            <artifactId>pebble</artifactId>
+            <version>3.1.0</version>
+        </dependency>
+
+        <!-- Google Cloud -->
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-bigquery</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-storage</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.auto.value</groupId>
+            <artifactId>auto-value-annotations</artifactId>
+            <version>1.6.6</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.auto.value</groupId>
+            <artifactId>auto-value</artifactId>
+            <version>1.6.6</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.12</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.beam</groupId>
+            <artifactId>beam-runners-direct-java</artifactId>
+            <version>${org.apache.beam.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest-library</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.beam</groupId>
+            <artifactId>beam-sdks-java-io-google-cloud-platform</artifactId>
+            <version>2.16.0</version>
+            <scope>compile</scope>
+        </dependency>
+    </dependencies>
+
+</project>
diff --git a/.../connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/common/TypeUtil.java b/.../connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/common/TypeUtil.java
@@ -0,0 +1,66 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.storage.connectors.bigquery.common;
+
+import com.google.cloud.bigquery.StandardSQLTypeName;
+import feast.types.ValueProto;
+import java.util.HashMap;
+import java.util.Map;
+
+public class TypeUtil {
+
+  private static final Map<ValueProto.ValueType.Enum, StandardSQLTypeName>
+      VALUE_TYPE_TO_STANDARD_SQL_TYPE = new HashMap<>();
+
+  static {
+    VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(ValueProto.ValueType.Enum.BYTES, StandardSQLTypeName.BYTES);
+    VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(
+        ValueProto.ValueType.Enum.STRING, StandardSQLTypeName.STRING);
+    VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(ValueProto.ValueType.Enum.INT32, StandardSQLTypeName.INT64);
+    VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(ValueProto.ValueType.Enum.INT64, StandardSQLTypeName.INT64);
+    VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(
+        ValueProto.ValueType.Enum.DOUBLE, StandardSQLTypeName.FLOAT64);
+    VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(
+        ValueProto.ValueType.Enum.FLOAT, StandardSQLTypeName.FLOAT64);
+    VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(ValueProto.ValueType.Enum.BOOL, StandardSQLTypeName.BOOL);
+    VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(
+        ValueProto.ValueType.Enum.BYTES_LIST, StandardSQLTypeName.BYTES);
+    VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(
+        ValueProto.ValueType.Enum.STRING_LIST, StandardSQLTypeName.STRING);
+    VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(
+        ValueProto.ValueType.Enum.INT32_LIST, StandardSQLTypeName.INT64);
+    VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(
+        ValueProto.ValueType.Enum.INT64_LIST, StandardSQLTypeName.INT64);
+    VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(
+        ValueProto.ValueType.Enum.DOUBLE_LIST, StandardSQLTypeName.FLOAT64);
+    VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(
+        ValueProto.ValueType.Enum.FLOAT_LIST, StandardSQLTypeName.FLOAT64);
+    VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(
+        ValueProto.ValueType.Enum.BOOL_LIST, StandardSQLTypeName.BOOL);
+  }
+
+  /**
+   * Converts {@link feast.types.ValueProto.ValueType} to its corresponding {@link
+   * StandardSQLTypeName}
+   *
+   * @param valueType value type to convert
+   * @return {@link StandardSQLTypeName}
+   */
+  public static StandardSQLTypeName toStandardSqlType(ValueProto.ValueType.Enum valueType) {
+    return VALUE_TYPE_TO_STANDARD_SQL_TYPE.get(valueType);
+  }
+}