Skip to content

Commit

Permalink
feat(codec): encode map type into row
Browse files Browse the repository at this point in the history
  • Loading branch information
@aceforeverd
aceforeverd committed Jan 22, 2024
1 parent 9fe51b9 commit 6cc89a2
Show file tree
Hide file tree
Showing 38 changed files with 1,897 additions and 887 deletions.
30 changes: 30 additions & 0 deletions cases/query/udf_query.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -588,3 +588,33 @@ cases:
columns: ["e1 bool", "e2 bool", "e3 bool"]
data: |
true, false, true
- id: 15
mode: request-unsupport
sql: |
select map(1, 2, 3, 4) as c1
- id: 16
mode: request-unsupport
# this covers basic codec for map data type
sql: |
select
c1[3] as o1, c2[1] as o2, c3['6'] as o3, c4[timestamp(8000)] as o4,
c5[int64(12)] as o5
from (select
map(1, 2, 3, 4) as c1,
map(1, '2', 3, '4') as c2,
map('5', timestamp(8000), '6', timestamp(9000)) as c3,
map(timestamp(8000), date("2012-12-12"), timestamp(9000), date("2014-11-11")) as c4,
map(int64(10), int16(11), int64(12), int16(13)) as c5
)
expect:
columns: ["o1 int", "o2 string", "o3 timestamp", "o4 date", "o5 int16"]
data: |
4, 2, 9000, 2012-12-12, 13
- id: 17
mode: request-unsupport
sql: |
select c1 + 8 from (select 9 as c1)
2 changes: 1 addition & 1 deletion hybridse/examples/toydb/src/tablet/tablet_catalog.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ bool TabletTableHandler::Init() {
// init types var
for (int32_t i = 0; i < schema_.size(); i++) {
const type::ColumnDef& column = schema_.Get(i);
codec::ColInfo col_info(column.name(), column.type(), i, 0);
codec::ColInfo col_info(column.name(), column.schema(), i, 0);
types_.insert(std::make_pair(column.name(), col_info));
}

Expand Down
31 changes: 31 additions & 0 deletions hybridse/include/base/fe_status.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,37 @@ static inline std::initializer_list<int> __output_literal_args(STREAM& stream,

#define MAX_STATUS_TRACE_SIZE 4096

// Evaluate and check the expression returns a absl::Status.
// End the current function by return status, if status is not OK
#define CHECK_ABSL_STATUS(expr) \
while (true) { \
auto _s = (expr); \
if (!_s.ok()) { \
return _s; \
} \
break; \
}

// Check the absl::StatusOr<T> object, end the current function
// by return 'object.status()' if it is not OK
#define CHECK_ABSL_STATUSOR(statusor) \
while (true) { \
if (!statusor.ok()) { \
return statusor.status(); \
} \
break; \
}

// Evaluate the expression returns Status, converted and return failed absl status if status not ok
#define CHECK_STATUS_TO_ABSL(expr) \
while (true) { \
auto _status = (expr); \
if (!_status.isOK()) { \
return absl::InternalError(_status.GetMsg()); \
} \
break; \
}

#define CHECK_STATUS(call, ...) \
while (true) { \
auto _status = (call); \
Expand Down
11 changes: 10 additions & 1 deletion hybridse/include/codec/fe_row_codec.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,18 @@ static constexpr uint32_t UINT24_MAX = (1 << 24) - 1;
const std::string NONETOKEN = "!N@U#L$L%"; // NOLINT
const std::string EMPTY_STRING = "!@#$%"; // NOLINT

// TODO(chendihao): Change to inline function if do not depend on gflags
const std::unordered_map<::hybridse::type::Type, uint8_t>& GetTypeSizeMap();

// return true if the column considered base type in row codec.
// date & timestamp consider base type since they have single field in corresponding llvm struct,
// while string, map and array consider complex type.
//
// for base types, the column is written into row ptr by just writing the value of primitive type,
// for comple type, written is made by a string (or string-like) manner: str size + str data.
// map, array, or any other complex types, takes a extra encoding from their struct value into str data.
bool IsCodecBaseType(const type::ColumnSchema& sc);
bool IsCodecStrLikeType(const type::ColumnSchema& sc);

inline uint8_t GetAddrLength(uint32_t size) {
if (size <= UINT8_MAX) {
return 1;
Expand Down
3 changes: 3 additions & 0 deletions hybridse/include/codec/type_codec.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,9 @@ int32_t AppendString(int8_t* buf_ptr, uint32_t buf_size, uint32_t col_idx,
uint32_t str_start_offset, uint32_t str_field_offset,
uint32_t str_addr_space, uint32_t str_body_offset);

// write `str_offset` in address `str_offset_ptr`, actual written bytes determined by `str_addr_space`
void EncodeStrOffset(int8_t* str_offset_ptr, int32_t str_offset, int32_t str_addr_space);

inline int8_t GetAddrSpace(uint32_t size) {
if (size <= UINT8_MAX) {
return 1;
Expand Down
24 changes: 24 additions & 0 deletions hybridse/src/case/sql_case_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,44 +31,51 @@ TEST_F(SqlCaseTest, ExtractTableDefTest) {
{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kVarchar);
column->mutable_schema()->set_base_type(::hybridse::type::kVarchar);
column->set_name("col0");
column->set_is_not_null(false);
}
{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kInt32);
column->mutable_schema()->set_base_type(::hybridse::type::kInt32);
column->set_name("col1");
column->set_is_not_null(false);
}
{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kInt16);
column->mutable_schema()->set_base_type(::hybridse::type::kInt16);
column->set_name("col2");
column->set_is_not_null(false);
}
{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kFloat);
column->mutable_schema()->set_base_type(::hybridse::type::kFloat);
column->set_name("col3");
column->set_is_not_null(false);
}
{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kDouble);
column->mutable_schema()->set_base_type(::hybridse::type::kDouble);
column->set_name("col4");
column->set_is_not_null(false);
}

{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kInt64);
column->mutable_schema()->set_base_type(::hybridse::type::kInt64);
column->set_name("col5");
column->set_is_not_null(false);
}

{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kVarchar);
column->mutable_schema()->set_base_type(::hybridse::type::kVarchar);
column->set_name("col6");
column->set_is_not_null(false);
}
Expand Down Expand Up @@ -142,18 +149,21 @@ TEST_F(SqlCaseTest, ExtractTableDefTest) {
{
::hybridse::type::ColumnDef* column = table2.add_columns();
column->set_type(::hybridse::type::kVarchar);
column->mutable_schema()->set_base_type(::hybridse::type::kVarchar);
column->set_name("col0");
column->set_is_not_null(false);
}
{
::hybridse::type::ColumnDef* column = table2.add_columns();
column->set_type(::hybridse::type::kInt32);
column->mutable_schema()->set_base_type(::hybridse::type::kInt32);
column->set_name("col1");
column->set_is_not_null(false);
}
{
::hybridse::type::ColumnDef* column = table2.add_columns();
column->set_type(::hybridse::type::kTimestamp);
column->mutable_schema()->set_base_type(::hybridse::type::kTimestamp);
column->set_name("coltime");
column->set_is_not_null(false);
}
Expand Down Expand Up @@ -448,44 +458,51 @@ TEST_F(SqlCaseTest, ExtractSqlCase) {
{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kVarchar);
column->mutable_schema()->set_base_type(::hybridse::type::kVarchar);
column->set_name("col0");
column->set_is_not_null(false);
}
{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kInt32);
column->mutable_schema()->set_base_type(::hybridse::type::kInt32);
column->set_name("col1");
column->set_is_not_null(false);
}
{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kInt16);
column->mutable_schema()->set_base_type(::hybridse::type::kInt16);
column->set_name("col2");
column->set_is_not_null(false);
}
{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kFloat);
column->mutable_schema()->set_base_type(::hybridse::type::kFloat);
column->set_name("col3");
column->set_is_not_null(false);
}
{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kDouble);
column->mutable_schema()->set_base_type(::hybridse::type::kDouble);
column->set_name("col4");
column->set_is_not_null(false);
}

{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kInt64);
column->mutable_schema()->set_base_type(::hybridse::type::kInt64);
column->set_name("col5");
column->set_is_not_null(false);
}

{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kVarchar);
column->mutable_schema()->set_base_type(::hybridse::type::kVarchar);
column->set_name("col6");
column->set_is_not_null(false);
}
Expand Down Expand Up @@ -613,43 +630,50 @@ TEST_F(SqlCaseTest, ExtractSqlCase) {
{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kVarchar);
column->mutable_schema()->set_base_type(::hybridse::type::kVarchar);
column->set_name("f0");
column->set_is_not_null(false);
}
{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kFloat);
column->mutable_schema()->set_base_type(::hybridse::type::kFloat);
column->set_name("f1");
column->set_is_not_null(false);
}
{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kDouble);
column->mutable_schema()->set_base_type(::hybridse::type::kDouble);
column->set_name("f2");
column->set_is_not_null(false);
}
{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kInt16);
column->mutable_schema()->set_base_type(::hybridse::type::kInt16);
column->set_name("f3");
column->set_is_not_null(false);
}
{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kInt32);
column->mutable_schema()->set_base_type(::hybridse::type::kInt32);
column->set_name("f4");
column->set_is_not_null(false);
}
{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kInt64);
column->mutable_schema()->set_base_type(::hybridse::type::kInt64);
column->set_name("f5");
column->set_is_not_null(false);
}

{
::hybridse::type::ColumnDef* column = table.add_columns();
column->set_type(::hybridse::type::kTimestamp);
column->mutable_schema()->set_base_type(::hybridse::type::kTimestamp);
column->set_name("f6");
column->set_is_not_null(false);
}
Expand Down
Loading

0 comments on commit 6cc89a2

Please sign in to comment.