Skip to content

Commit 26ae450

Browse files
vitalifvitalif
authored
Add StartTime/EndTime/UserSID for BuildIndex (#16971) (#17751)
1 parent d456fe9 commit 26ae450

File tree

18 files changed

+155
-20
lines changed

18 files changed

+155
-20
lines changed

‎ydb/core/grpc_services/operation_helpers.cpp

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,15 @@ Ydb::TOperationId ToOperationId(const NKikimrIndexBuilder::TIndexBuild& build) {
5959
void ToOperation(const NKikimrIndexBuilder::TIndexBuild& build, Ydb::Operations::Operation* operation) {
6060
operation->set_id(NOperationId::ProtoToString(ToOperationId(build)));
6161
operation->mutable_issues()->CopyFrom(build.GetIssues());
62+
if (build.HasStartTime()) {
63+
*operation->mutable_create_time() = build.GetStartTime();
64+
}
65+
if (build.HasEndTime()) {
66+
*operation->mutable_end_time() = build.GetEndTime();
67+
}
68+
if (build.HasUserSID()) {
69+
operation->set_created_by(build.GetUserSID());
70+
}
6271

6372
switch (build.GetState()) {
6473
case Ydb::Table::IndexBuildState::STATE_DONE:

‎ydb/core/grpc_services/rpc_alter_table.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,9 @@ class TAlterTableRPC : public TRpcSchemeRequestActor<TAlterTableRPC, TEvAlterTab
342342
void SendAddIndexOpToSS(const TActorContext& ctx, ui64 schemeShardId) {
343343
SetSchemeShardId(schemeShardId);
344344
auto ev = std::make_unique<NSchemeShard::TEvIndexBuilder::TEvCreateRequest>(TxId, DatabaseName, std::move(IndexBuildSettings));
345+
if (UserToken) {
346+
ev->Record.SetUserSID(UserToken->GetUserSID());
347+
}
345348
ForwardToSchemeShard(ctx, std::move(ev));
346349
}
347350

‎ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -697,6 +697,9 @@ class TKqpSchemeExecuter : public TActorBootstrapped<TKqpSchemeExecuter> {
697697
const auto& buildOp = schemeOp.GetBuildOperation();
698698
SetSchemeShardId(domainInfo->ExtractSchemeShard());
699699
auto req = std::make_unique<NSchemeShard::TEvIndexBuilder::TEvCreateRequest>(TxId, Database, buildOp);
700+
if (UserToken) {
701+
req->Record.SetUserSID(UserToken->GetUserSID());
702+
}
700703
ForwardToSchemeShard(std::move(req));
701704
}
702705

‎ydb/core/kqp/ut/indexes/kqp_indexes_vector_ut.cpp

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include <ydb/core/kqp/gateway/kqp_metadata_loader.h>
77
#include <ydb/core/kqp/host/kqp_host_impl.h>
88

9+
#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/operation/operation.h>
910
#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/proto/accessor.h>
1011
#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/table.h>
1112

@@ -609,6 +610,52 @@ Y_UNIT_TEST_SUITE(KqpVectorIndexes) {
609610
// DoPositiveQueriesVectorIndexOrderByCosine(session);
610611
}
611612

613+
Y_UNIT_TEST(BuildIndexTimesAndUser) {
614+
NKikimrConfig::TAppConfig appConfig;
615+
NKikimrConfig::TFeatureFlags featureFlags;
616+
featureFlags.SetEnableVectorIndex(true);
617+
auto setting = NKikimrKqp::TKqpSetting();
618+
auto serverSettings = TKikimrSettings()
619+
.SetAppConfig(appConfig)
620+
.SetFeatureFlags(featureFlags)
621+
.SetKqpSettings({setting});
622+
623+
TKikimrRunner kikimr(serverSettings);
624+
625+
auto now = TInstant::Now();
626+
627+
auto driver = NYdb::TDriver(NYdb::TDriverConfig()
628+
.SetEndpoint(kikimr.GetEndpoint())
629+
.SetDatabase("/Root")
630+
.SetAuthToken("root@builtin"));
631+
auto db = NYdb::NTable::TTableClient(driver);
632+
auto session = DoCreateTableForVectorIndex(db, false);
633+
{
634+
const TString createIndex(Q_(R"(
635+
ALTER TABLE `/Root/TestTable`
636+
ADD INDEX index
637+
GLOBAL USING vector_kmeans_tree
638+
ON (emb)
639+
WITH (distance=cosine, vector_type="uint8", vector_dimension=2, levels=2, clusters=2);
640+
)"));
641+
642+
auto result = session.ExecuteSchemeQuery(createIndex)
643+
.ExtractValueSync();
644+
645+
UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
646+
}
647+
{
648+
NYdb::NOperation::TOperationClient client(kikimr.GetDriver());
649+
auto list = client.List<NYdb::NTable::TBuildIndexOperation>().ExtractValueSync();
650+
UNIT_ASSERT_EQUAL(list.GetList().size(), 1);
651+
auto & op = list.GetList()[0];
652+
UNIT_ASSERT_EQUAL(op.Status().GetStatus(), NYdb::EStatus::SUCCESS);
653+
UNIT_ASSERT(op.CreateTime() >= TInstant::Seconds(now.Seconds()));
654+
UNIT_ASSERT(op.EndTime() >= op.CreateTime());
655+
UNIT_ASSERT_EQUAL(op.CreatedBy(), "root@builtin");
656+
}
657+
}
658+
612659
Y_UNIT_TEST(VectorIndexIsNotUpdatable) {
613660
NKikimrConfig::TFeatureFlags featureFlags;
614661
featureFlags.SetEnableVectorIndex(true);

‎ydb/core/protos/index_builder.proto

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import "ydb/public/api/protos/ydb_status_codes.proto";
44
import "ydb/public/api/protos/ydb_table.proto";
55
import "ydb/public/api/protos/ydb_value.proto";
66
import "google/protobuf/any.proto";
7+
import "google/protobuf/timestamp.proto";
78

89

910
package NKikimrIndexBuilder;
@@ -51,6 +52,9 @@ message TIndexBuild {
5152
optional Ydb.Table.IndexBuildState.State State = 3;
5253
optional TIndexBuildSettings Settings = 4;
5354
optional float Progress = 5 [default = 0];
55+
optional google.protobuf.Timestamp StartTime = 6;
56+
optional google.protobuf.Timestamp EndTime = 7;
57+
optional string UserSID = 8;
5458
}
5559

5660
message TEvCreateRequest {
@@ -60,6 +64,7 @@ message TEvCreateRequest {
6064
optional TIndexBuildSettings Settings = 4;
6165
// Internal flag is true for system-generated operations and is false for those initiated directly by the user.
6266
optional bool Internal = 5 [default = false];
67+
optional string UserSID = 6;
6368
}
6469

6570
message TEvCreateResponse {

‎ydb/core/tx/schemeshard/schemeshard__init.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3410,7 +3410,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
34103410
}
34113411
}
34123412

3413-
// Read KesusAlters
3413+
// Read KesusAlters
34143414
{
34153415
TKesusAlterRows kesusAlterRows;
34163416
if (!LoadKesusAlters(db, kesusAlterRows)) {

‎ydb/core/tx/schemeshard/schemeshard_build_index.cpp

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,14 @@ void TSchemeShard::PersistCreateBuildIndex(NIceDb::TNiceDb& db, const TIndexBuil
6767
NIceDb::TUpdate<Schema::IndexBuild::MaxBatchBytes>(info.ScanSettings.GetMaxBatchBytes()),
6868
NIceDb::TUpdate<Schema::IndexBuild::MaxShards>(info.MaxInProgressShards),
6969
NIceDb::TUpdate<Schema::IndexBuild::MaxRetries>(info.ScanSettings.GetMaxBatchRetries()),
70-
NIceDb::TUpdate<Schema::IndexBuild::BuildKind>(ui32(info.BuildKind))
70+
NIceDb::TUpdate<Schema::IndexBuild::BuildKind>(ui32(info.BuildKind)),
71+
NIceDb::TUpdate<Schema::IndexBuild::StartTime>(info.StartTime.Seconds())
7172
);
73+
if (info.UserSID) {
74+
persistedBuildIndex.Update(
75+
NIceDb::TUpdate<Schema::IndexBuild::UserSID>(*info.UserSID)
76+
);
77+
}
7278
// Persist details of the index build operation: ImplTableDescriptions and SpecializedIndexDescription.
7379
// We have chosen TIndexCreationConfig's string representation as the serialization format.
7480
if (bool hasSpecializedDescription = !std::holds_alternative<std::monostate>(info.SpecializedIndexDescription);
@@ -119,7 +125,11 @@ void TSchemeShard::PersistCreateBuildIndex(NIceDb::TNiceDb& db, const TIndexBuil
119125

120126
void TSchemeShard::PersistBuildIndexState(NIceDb::TNiceDb& db, const TIndexBuildInfo& indexInfo) {
121127
db.Table<Schema::IndexBuild>().Key(indexInfo.Id).Update(
122-
NIceDb::TUpdate<Schema::IndexBuild::State>(ui32(indexInfo.State)));
128+
NIceDb::TUpdate<Schema::IndexBuild::State>(ui32(indexInfo.State)),
129+
NIceDb::TUpdate<Schema::IndexBuild::Issue>(indexInfo.Issue),
130+
NIceDb::TUpdate<Schema::IndexBuild::StartTime>(indexInfo.StartTime.Seconds()),
131+
NIceDb::TUpdate<Schema::IndexBuild::EndTime>(indexInfo.EndTime.Seconds())
132+
);
123133
}
124134

125135
void TSchemeShard::PersistBuildIndexCancelRequest(NIceDb::TNiceDb& db, const TIndexBuildInfo& indexInfo) {

‎ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,10 @@ class TSchemeShard::TIndexBuilder::TTxCreate: public TSchemeShard::TIndexBuilder
182182

183183
buildInfo->CreateSender = Request->Sender;
184184
buildInfo->SenderCookie = Request->Cookie;
185+
buildInfo->StartTime = TAppData::TimeProvider->Now();
186+
if (request.HasUserSID()) {
187+
buildInfo->UserSID = request.GetUserSID();
188+
}
185189

186190
Self->PersistCreateBuildIndex(db, *buildInfo);
187191

‎ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,11 @@ void TSchemeShard::TIndexBuilder::TTxBase::ApplyState(NTabletFlatExecutor::TTran
2323
Y_VERIFY_S(buildInfoPtr, "IndexBuilds has no " << buildId);
2424
auto& buildInfo = *buildInfoPtr->Get();
2525
LOG_I("Change state from " << buildInfo.State << " to " << state);
26+
if (state == TIndexBuildInfo::EState::Rejected ||
27+
state == TIndexBuildInfo::EState::Cancelled ||
28+
state == TIndexBuildInfo::EState::Done) {
29+
buildInfo.EndTime = TAppData::TimeProvider->Now();
30+
}
2631
buildInfo.State = state;
2732

2833
NIceDb::TNiceDb db(txc.DB);
@@ -195,6 +200,15 @@ void TSchemeShard::TIndexBuilder::TTxBase::Fill(NKikimrIndexBuilder::TIndexBuild
195200
if (indexInfo.Issue) {
196201
AddIssue(index.MutableIssues(), indexInfo.Issue);
197202
}
203+
if (indexInfo.StartTime != TInstant::Zero()) {
204+
*index.MutableStartTime() = SecondsToProtoTimeStamp(indexInfo.StartTime.Seconds());
205+
}
206+
if (indexInfo.EndTime != TInstant::Zero()) {
207+
*index.MutableEndTime() = SecondsToProtoTimeStamp(indexInfo.EndTime.Seconds());
208+
}
209+
if (indexInfo.UserSID) {
210+
index.SetUserSID(*indexInfo.UserSID);
211+
}
198212

199213
for (const auto& item: indexInfo.Shards) {
200214
const TShardIdx& shardIdx = item.first;

‎ydb/core/tx/schemeshard/schemeshard_export.cpp

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,6 @@ namespace {
2929
}
3030
}
3131

32-
NProtoBuf::Timestamp SecondsToProtoTimeStamp(ui64 sec) {
33-
NProtoBuf::Timestamp timestamp;
34-
timestamp.set_seconds((i64)(sec));
35-
timestamp.set_nanos(0);
36-
return timestamp;
37-
}
38-
3932
void FillItemProgress(TSchemeShard* ss, const TExportInfo::TPtr exportInfo, ui32 itemIdx,
4033
Ydb::Export::ExportItemProgress& itemProgress) {
4134

0 commit comments

Comments
 (0)