Skip to content

Commit

Permalink
Restore indexes from backup with the original partitioning (#7589)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jepett0
jepett0 authored Aug 21, 2024
1 parent 7f8f368 commit e56caba
Show file tree
Hide file tree
Showing 7 changed files with 516 additions and 333 deletions.
1 change: 1 addition & 0 deletions ydb/core/protos/flat_scheme_op.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1855,6 +1855,7 @@ message TDescribeOptions {
optional bool ReturnChannelsBinding = 8 [default = false];
optional bool ReturnRangeKey = 9 [default = true];
optional bool ReturnSetVal = 10 [default = false];
optional bool ReturnIndexTableBoundaries = 11 [default = false];
}

// Request to read scheme for a specific path
Expand Down
1 change: 1 addition & 0 deletions ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ static NKikimrSchemeOp::TPathDescription GetTableDescription(TSchemeShard* ss, c
opts.SetReturnPartitioningInfo(false);
opts.SetReturnPartitionConfig(true);
opts.SetReturnBoundaries(true);
opts.SetReturnIndexTableBoundaries(true);

auto desc = DescribePath(ss, TlsActivationContext->AsActorContext(), pathId, opts);
auto record = desc->GetRecord();
Expand Down
142 changes: 82 additions & 60 deletions ydb/core/tx/schemeshard/schemeshard_path_describer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,79 @@ static void FillTableStats(NKikimrSchemeOp::TPathDescription& pathDescription, c
FillTableMetrics(pathDescription.MutableTabletMetrics(), stats);
}

static void FillColumns(
const TTableInfo& tableInfo,
google::protobuf::RepeatedPtrField<NKikimrSchemeOp::TColumnDescription>& out
) {
bool familyNamesBuilt = false;
THashMap<ui32, TString> familyNames;

out.Reserve(tableInfo.Columns.size());
for (const auto& col : tableInfo.Columns) {
const auto& cinfo = col.second;
if (cinfo.IsDropped())
continue;

auto* colDescr = out.Add();
colDescr->SetName(cinfo.Name);
colDescr->SetType(NScheme::TypeName(cinfo.PType, cinfo.PTypeMod));
auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod);
colDescr->SetTypeId(columnType.TypeId);
if (columnType.TypeInfo) {
*colDescr->MutableTypeInfo() = *columnType.TypeInfo;
}
colDescr->SetId(cinfo.Id);
colDescr->SetNotNull(cinfo.NotNull);

if (cinfo.Family != 0) {
colDescr->SetFamily(cinfo.Family);

if (!familyNamesBuilt) {
for (const auto& family : tableInfo.PartitionConfig().GetColumnFamilies()) {
if (family.HasName() && family.HasId()) {
familyNames[family.GetId()] = family.GetName();
}
}
familyNamesBuilt = true;
}

auto it = familyNames.find(cinfo.Family);
if (it != familyNames.end() && !it->second.empty()) {
colDescr->SetFamilyName(it->second);
}
}

colDescr->SetIsBuildInProgress(cinfo.IsBuildInProgress);

switch (cinfo.DefaultKind) {
case ETableColumnDefaultKind::None:
break;
case ETableColumnDefaultKind::FromSequence:
colDescr->SetDefaultFromSequence(cinfo.DefaultValue);
break;
case ETableColumnDefaultKind::FromLiteral:
Y_ABORT_UNLESS(colDescr->MutableDefaultFromLiteral()->ParseFromString(
cinfo.DefaultValue));
break;
}
}
}

static void FillKeyColumns(
const TTableInfo& tableInfo,
google::protobuf::RepeatedPtrField<TProtoStringType>& names,
google::protobuf::RepeatedField<ui32>& ids
) {
Y_ABORT_UNLESS(!tableInfo.KeyColumnIds.empty());
names.Reserve(tableInfo.KeyColumnIds.size());
ids.Reserve(tableInfo.KeyColumnIds.size());
for (ui32 keyColId : tableInfo.KeyColumnIds) {
*names.Add() = tableInfo.Columns.at(keyColId).Name;
*ids.Add() = keyColId;
}

}

void TPathDescriber::FillPathDescr(NKikimrSchemeOp::TDirEntry* descr, TPathElement::TPtr pathEl, TPathElement::EPathSubType subType) {
FillChildDescr(descr, pathEl);

Expand Down Expand Up @@ -303,6 +376,7 @@ void TPathDescriber::DescribeTable(const TActorContext& ctx, TPathId pathId, TPa
bool returnBoundaries = false;
bool returnRangeKey = true;
bool returnSetVal = Params.GetOptions().GetReturnSetVal();
bool returnIndexTableBoundaries = Params.GetOptions().GetReturnIndexTableBoundaries();
if (Params.HasOptions()) {
returnConfig = Params.GetOptions().GetReturnPartitionConfig();
returnPartitioning = Params.GetOptions().GetReturnPartitioningInfo();
Expand Down Expand Up @@ -427,7 +501,9 @@ void TPathDescriber::DescribeTable(const TActorContext& ctx, TPathId pathId, TPa

switch (childPath->PathType) {
case NKikimrSchemeOp::EPathTypeTableIndex:
Self->DescribeTableIndex(childPathId, childName, returnConfig, false, *entry->AddTableIndexes());
Self->DescribeTableIndex(
childPathId, childName, returnConfig, returnIndexTableBoundaries, *entry->AddTableIndexes()
);
break;
case NKikimrSchemeOp::EPathTypeCdcStream:
Self->DescribeCdcStream(childPathId, childName, *entry->AddCdcStreams());
Expand Down Expand Up @@ -1189,67 +1265,10 @@ void TSchemeShard::DescribeTable(
) const
{
Y_UNUSED(typeRegistry);
THashMap<ui32, TString> familyNames;
bool familyNamesBuilt = false;

entry->SetTableSchemaVersion(tableInfo.AlterVersion);
entry->MutableColumns()->Reserve(tableInfo.Columns.size());
for (auto col : tableInfo.Columns) {
const auto& cinfo = col.second;
if (cinfo.IsDropped())
continue;

auto colDescr = entry->AddColumns();
colDescr->SetName(cinfo.Name);
colDescr->SetType(NScheme::TypeName(cinfo.PType, cinfo.PTypeMod));
auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod);
colDescr->SetTypeId(columnType.TypeId);
if (columnType.TypeInfo) {
*colDescr->MutableTypeInfo() = *columnType.TypeInfo;
}
colDescr->SetId(cinfo.Id);
colDescr->SetNotNull(cinfo.NotNull);

if (cinfo.Family != 0) {
colDescr->SetFamily(cinfo.Family);

if (!familyNamesBuilt) {
for (const auto& family : tableInfo.PartitionConfig().GetColumnFamilies()) {
if (family.HasName() && family.HasId()) {
familyNames[family.GetId()] = family.GetName();
}
}
familyNamesBuilt = true;
}

auto it = familyNames.find(cinfo.Family);
if (it != familyNames.end() && !it->second.empty()) {
colDescr->SetFamilyName(it->second);
}
}

colDescr->SetIsBuildInProgress(cinfo.IsBuildInProgress);

switch (cinfo.DefaultKind) {
case ETableColumnDefaultKind::None:
break;
case ETableColumnDefaultKind::FromSequence:
colDescr->SetDefaultFromSequence(cinfo.DefaultValue);
break;
case ETableColumnDefaultKind::FromLiteral:
Y_ABORT_UNLESS(colDescr->MutableDefaultFromLiteral()->ParseFromString(
cinfo.DefaultValue));
break;
}
}
Y_ABORT_UNLESS(!tableInfo.KeyColumnIds.empty());

entry->MutableKeyColumnNames()->Reserve(tableInfo.KeyColumnIds.size());
entry->MutableKeyColumnIds()->Reserve(tableInfo.KeyColumnIds.size());
for (ui32 keyColId : tableInfo.KeyColumnIds) {
entry->AddKeyColumnNames(tableInfo.Columns.at(keyColId).Name);
entry->AddKeyColumnIds(keyColId);
}
FillColumns(tableInfo, *entry->MutableColumns());
FillKeyColumns(tableInfo, *entry->MutableKeyColumnNames(), *entry->MutableKeyColumnIds());

if (fillConfig) {
FillPartitionConfig(tableInfo.PartitionConfig(), *entry->MutablePartitionConfig());
Expand Down Expand Up @@ -1328,6 +1347,9 @@ void TSchemeShard::DescribeTableIndex(const TPathId& pathId, const TString& name
FillPartitionConfig(tableInfo.PartitionConfig(), *tableDescription->MutablePartitionConfig());
}
if (fillBoundaries) {
// column info is necessary for split boundary type conversion
FillColumns(tableInfo, *tableDescription->MutableColumns());
FillKeyColumns(tableInfo, *tableDescription->MutableKeyColumnNames(), *tableDescription->MutableKeyColumnIds());
FillTableBoundaries(tableDescription->MutableSplitBoundary(), tableInfo);
}
}
Expand Down
4 changes: 2 additions & 2 deletions ydb/core/ydb_convert/table_description.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -852,8 +852,8 @@ void FillGlobalIndexSettings(Ydb::Table::GlobalIndexSettings& settings,
NKikimrMiniKQL::TType splitKeyType;
Ydb::Table::DescribeTableResult unused;
FillColumnDescription(unused, splitKeyType, indexImplTableDescription);
FillTableBoundaryImpl(
*settings.mutable_partition_at_keys(),
FillTableBoundaryImpl<Ydb::Table::GlobalIndexSettings>(
settings,
indexImplTableDescription,
splitKeyType
);
Expand Down
21 changes: 17 additions & 4 deletions ydb/public/sdk/cpp/client/ydb_table/table.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -469,6 +469,10 @@ class TTableDescription::TImpl {
Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, dataColumns));
}

void AddSecondaryIndex(const TIndexDescription& indexDescription) {
Indexes_.emplace_back(indexDescription);
}

void AddVectorIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVectorIndexSettings& vectorIndexSettings) {
Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, {}, {}, vectorIndexSettings));
}
Expand Down Expand Up @@ -749,6 +753,10 @@ void TTableDescription::AddSecondaryIndex(const TString& indexName, EIndexType t
Impl_->AddSecondaryIndex(indexName, type, indexColumns, dataColumns);
}

void TTableDescription::AddSecondaryIndex(const TIndexDescription& indexDescription) {
Impl_->AddSecondaryIndex(indexDescription);
}

void TTableDescription::AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns) {
AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns);
}
Expand Down Expand Up @@ -1173,6 +1181,11 @@ TTableBuilder& TTableBuilder::SetPrimaryKeyColumn(const TString& primaryKeyColum
return *this;
}

TTableBuilder& TTableBuilder::AddSecondaryIndex(const TIndexDescription& indexDescription) {
TableDescription_.AddSecondaryIndex(indexDescription);
return *this;
}

TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) {
TableDescription_.AddSecondaryIndex(indexName, type, indexColumns, dataColumns);
return *this;
Expand Down Expand Up @@ -2364,7 +2377,7 @@ TVectorIndexSettings TVectorIndexSettings::FromProto(const TProto& proto) {
default:
return EVectorType::Unknown;
}
};
};


auto metricFromProto = [&](const auto& proto) -> TVectorIndexSettings::TMetric {
Expand All @@ -2376,7 +2389,7 @@ TVectorIndexSettings TVectorIndexSettings::FromProto(const TProto& proto) {
default:
return {};
}
};
};

return {
.Metric = metricFromProto(proto),
Expand Down Expand Up @@ -2424,8 +2437,8 @@ void TVectorIndexSettings::SerializeTo(Ydb::Table::VectorIndexSettings& settings
return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_UNSPECIFIED;
}
};


if (const auto* distance = std::get_if<EDistance>(&Metric)) {
settings.set_distance(convertDistance(*distance));
} else if (const auto* similarity = std::get_if<ESimilarity>(&Metric)) {
Expand Down
6 changes: 4 additions & 2 deletions ydb/public/sdk/cpp/client/ydb_table/table.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,10 +174,10 @@ struct TExplicitPartitions {
using TSelf = TExplicitPartitions;

FLUENT_SETTING_VECTOR(TValue, SplitPoints);

template <typename TProto>
static TExplicitPartitions FromProto(const TProto& proto);

void SerializeTo(Ydb::Table::ExplicitPartitions& proto) const;
};

Expand Down Expand Up @@ -642,6 +642,7 @@ class TTableDescription {
// common
void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns);
void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns);
void AddSecondaryIndex(const TIndexDescription& indexDescription);
// sync
void AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns);
void AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns);
Expand Down Expand Up @@ -855,6 +856,7 @@ class TTableBuilder {
TTableBuilder& SetPrimaryKeyColumn(const TString& primaryKeyColumn);

// common
TTableBuilder& AddSecondaryIndex(const TIndexDescription& indexDescription);
TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns);
TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns);
TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TString& indexColumn);
Expand Down
Loading

0 comments on commit e56caba

Please sign in to comment.