Skip to content

Commit e379e67

Browse files
n00bcrackern00bcracker
committed
Added sys view path type processing in SchemeCache
1 parent c2d2843 commit e379e67

File tree

6 files changed

+147
-41
lines changed

6 files changed

+147
-41
lines changed

‎ydb/core/protos/sys_view_types.proto

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,36 @@ option java_package = "ru.yandex.kikimr.proto";
44

55
enum ESysViewType {
66
EPartitionStats = 1;
7+
ENodes = 2;
8+
ETopQueriesByDurationOneMinute = 3;
9+
ETopQueriesByDurationOneHour = 4;
10+
ETopQueriesByReadBytesOneMinute = 5;
11+
ETopQueriesByReadBytesOneHour = 6;
12+
ETopQueriesByCpuTimeOneMinute = 7;
13+
ETopQueriesByCpuTimeOneHour = 8;
14+
ETopQueriesByRequestUnitsOneMinute = 9;
15+
ETopQueriesByRequestUnitsOneHour = 10;
16+
EQuerySessions = 11;
17+
EPDisks = 12;
18+
EVSlots = 13;
19+
EGroups = 14;
20+
EStoragePools = 15;
21+
EStorageStats = 16;
22+
ETablets = 17;
23+
EQueryMetricsOneMinute = 18;
24+
ETopPartitionsByCpuOneMinute = 19;
25+
ETopPartitionsByCpuOneHour = 20;
26+
ETopPartitionsByTliOneMinute = 21;
27+
ETopPartitionsByTliOneHour = 22;
28+
EResourcePoolClassifiers = 23;
29+
EResourcePools = 24;
30+
EAuthUsers = 25;
31+
EAuthGroups = 26;
32+
EAuthGroupMembers = 27;
33+
EAuthOwners = 28;
34+
EAuthPermissions = 29;
35+
EAuthEffectivePermissions = 30;
36+
EPgTables = 31;
37+
EInformationSchemaTables = 32;
38+
EPgClass = 33;
739
}

‎ydb/core/sys_view/common/schema.cpp

Lines changed: 57 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@
33
#include <ydb/core/base/appdata.h>
44
#include <yql/essentials/parser/pg_catalog/catalog.h>
55

6+
namespace {
7+
using NKikimrSysView::ESysViewType;
8+
}
9+
610
namespace NKikimr {
711
namespace NSysView {
812

@@ -170,6 +174,11 @@ class TSystemViewResolver : public ISystemViewResolver {
170174
return view ? TMaybe<TSchema>(*view) : Nothing();
171175
}
172176

177+
TMaybe<TSchema> GetSystemViewSchema(ESysViewType viewType) const override final {
178+
const TSchema* view = SystemViews.FindPtr(viewType);
179+
return view ? TMaybe<TSchema>(*view) : Nothing();
180+
}
181+
173182
TVector<TString> GetSystemViewNames(ETarget target) const override {
174183
TVector<TString> result;
175184
switch (target) {
@@ -210,41 +219,50 @@ class TSystemViewResolver : public ISystemViewResolver {
210219

211220
private:
212221
void RegisterPgTablesSystemViews() {
213-
auto registerView = [&](TStringBuf tableName, const TVector<Schema::PgColumn>& columns) {
222+
auto registerView = [&](TStringBuf tableName, ESysViewType type, const TVector<Schema::PgColumn>& columns) {
214223
auto& dsv = DomainSystemViews[tableName];
215224
auto& sdsv = SubDomainSystemViews[tableName];
225+
auto& sv = SystemViews[type];
216226
for (const auto& column : columns) {
217227
dsv.Columns[column._ColumnId + 1] = TSysTables::TTableColumnInfo(
218228
column._ColumnName, column._ColumnId + 1, column._ColumnTypeInfo, "", -1
219229
);
220230
sdsv.Columns[column._ColumnId + 1] = TSysTables::TTableColumnInfo(
221231
column._ColumnName, column._ColumnId + 1, column._ColumnTypeInfo, "", -1
222232
);
233+
sv.Columns[column._ColumnId + 1] = TSysTables::TTableColumnInfo(
234+
column._ColumnName, column._ColumnId + 1, column._ColumnTypeInfo, "", -1
235+
);
223236
}
224237
};
225238
registerView(
226239
PgTablesName,
240+
ESysViewType::EPgTables,
227241
Singleton<Schema::PgTablesSchemaProvider>()->GetColumns(PgTablesName)
228242
);
229243
registerView(
230244
InformationSchemaTablesName,
245+
ESysViewType::EInformationSchemaTables,
231246
Singleton<Schema::PgTablesSchemaProvider>()->GetColumns(InformationSchemaTablesName)
232247
);
233248
registerView(
234249
PgClassName,
250+
ESysViewType::EPgClass,
235251
Singleton<Schema::PgTablesSchemaProvider>()->GetColumns(PgClassName)
236252
);
237253
}
238254

239255
template <typename Table>
240-
void RegisterSystemView(const TStringBuf& name) {
256+
void RegisterSystemView(const TStringBuf& name, ESysViewType type) {
241257
TSchemaFiller<Table>::Fill(DomainSystemViews[name]);
242258
TSchemaFiller<Table>::Fill(SubDomainSystemViews[name]);
259+
TSchemaFiller<Table>::Fill(SystemViews[type]);
243260
}
244261

245262
template <typename Table>
246-
void RegisterDomainSystemView(const TStringBuf& name) {
263+
void RegisterDomainSystemView(const TStringBuf& name, ESysViewType type) {
247264
TSchemaFiller<Table>::Fill(DomainSystemViews[name]);
265+
TSchemaFiller<Table>::Fill(SystemViews[type]);
248266
}
249267

250268
template <typename Table>
@@ -258,29 +276,29 @@ class TSystemViewResolver : public ISystemViewResolver {
258276
}
259277

260278
void RegisterSystemViews() {
261-
RegisterSystemView<Schema::PartitionStats>(PartitionStatsName);
279+
RegisterSystemView<Schema::PartitionStats>(PartitionStatsName, ESysViewType::EPartitionStats);
262280

263-
RegisterSystemView<Schema::Nodes>(NodesName);
281+
RegisterSystemView<Schema::Nodes>(NodesName, ESysViewType::ENodes);
264282

265-
RegisterSystemView<Schema::QueryStats>(TopQueriesByDuration1MinuteName);
266-
RegisterSystemView<Schema::QueryStats>(TopQueriesByDuration1HourName);
267-
RegisterSystemView<Schema::QueryStats>(TopQueriesByReadBytes1MinuteName);
268-
RegisterSystemView<Schema::QueryStats>(TopQueriesByReadBytes1HourName);
269-
RegisterSystemView<Schema::QueryStats>(TopQueriesByCpuTime1MinuteName);
270-
RegisterSystemView<Schema::QueryStats>(TopQueriesByCpuTime1HourName);
271-
RegisterSystemView<Schema::QueryStats>(TopQueriesByRequestUnits1MinuteName);
272-
RegisterSystemView<Schema::QueryStats>(TopQueriesByRequestUnits1HourName);
273-
RegisterSystemView<Schema::QuerySessions>(QuerySessions);
283+
RegisterSystemView<Schema::QueryStats>(TopQueriesByDuration1MinuteName, ESysViewType::ETopQueriesByDurationOneMinute);
284+
RegisterSystemView<Schema::QueryStats>(TopQueriesByDuration1HourName, ESysViewType::ETopQueriesByDurationOneHour);
285+
RegisterSystemView<Schema::QueryStats>(TopQueriesByReadBytes1MinuteName, ESysViewType::ETopQueriesByReadBytesOneMinute);
286+
RegisterSystemView<Schema::QueryStats>(TopQueriesByReadBytes1HourName, ESysViewType::ETopQueriesByReadBytesOneHour);
287+
RegisterSystemView<Schema::QueryStats>(TopQueriesByCpuTime1MinuteName, ESysViewType::ETopQueriesByCpuTimeOneMinute);
288+
RegisterSystemView<Schema::QueryStats>(TopQueriesByCpuTime1HourName, ESysViewType::ETopQueriesByCpuTimeOneHour);
289+
RegisterSystemView<Schema::QueryStats>(TopQueriesByRequestUnits1MinuteName, ESysViewType::ETopQueriesByRequestUnitsOneMinute);
290+
RegisterSystemView<Schema::QueryStats>(TopQueriesByRequestUnits1HourName, ESysViewType::ETopQueriesByRequestUnitsOneHour);
291+
RegisterSystemView<Schema::QuerySessions>(QuerySessions, ESysViewType::EQuerySessions);
274292

275-
RegisterDomainSystemView<Schema::PDisks>(PDisksName);
276-
RegisterDomainSystemView<Schema::VSlots>(VSlotsName);
277-
RegisterDomainSystemView<Schema::Groups>(GroupsName);
278-
RegisterDomainSystemView<Schema::StoragePools>(StoragePoolsName);
279-
RegisterDomainSystemView<Schema::StorageStats>(StorageStatsName);
293+
RegisterDomainSystemView<Schema::PDisks>(PDisksName, ESysViewType::EPDisks);
294+
RegisterDomainSystemView<Schema::VSlots>(VSlotsName, ESysViewType::EVSlots);
295+
RegisterDomainSystemView<Schema::Groups>(GroupsName, ESysViewType::EGroups);
296+
RegisterDomainSystemView<Schema::StoragePools>(StoragePoolsName, ESysViewType::EStoragePools);
297+
RegisterDomainSystemView<Schema::StorageStats>(StorageStatsName, ESysViewType::EStorageStats);
280298

281-
RegisterDomainSystemView<Schema::Tablets>(TabletsName);
299+
RegisterDomainSystemView<Schema::Tablets>(TabletsName, ESysViewType::ETablets);
282300

283-
RegisterSystemView<Schema::QueryMetrics>(QueryMetricsName);
301+
RegisterSystemView<Schema::QueryMetrics>(QueryMetricsName, ESysViewType::EQueryMetricsOneMinute);
284302

285303
RegisterOlapStoreSystemView<Schema::PrimaryIndexStats>(StorePrimaryIndexStatsName);
286304
RegisterOlapStoreSystemView<Schema::PrimaryIndexPortionStats>(StorePrimaryIndexPortionStatsName);
@@ -291,24 +309,24 @@ class TSystemViewResolver : public ISystemViewResolver {
291309
RegisterColumnTableSystemView<Schema::PrimaryIndexGranuleStats>(TablePrimaryIndexGranuleStatsName);
292310
RegisterColumnTableSystemView<Schema::PrimaryIndexOptimizerStats>(TablePrimaryIndexOptimizerStatsName);
293311

294-
RegisterSystemView<Schema::TopPartitions>(TopPartitionsByCpu1MinuteName);
295-
RegisterSystemView<Schema::TopPartitions>(TopPartitionsByCpu1HourName);
296-
RegisterSystemView<Schema::TopPartitionsTli>(TopPartitionsByTli1MinuteName);
297-
RegisterSystemView<Schema::TopPartitionsTli>(TopPartitionsByTli1HourName);
312+
RegisterSystemView<Schema::TopPartitions>(TopPartitionsByCpu1MinuteName, ESysViewType::ETopPartitionsByCpuOneMinute);
313+
RegisterSystemView<Schema::TopPartitions>(TopPartitionsByCpu1HourName, ESysViewType::ETopPartitionsByCpuOneHour);
314+
RegisterSystemView<Schema::TopPartitionsTli>(TopPartitionsByTli1MinuteName, ESysViewType::ETopPartitionsByTliOneMinute);
315+
RegisterSystemView<Schema::TopPartitionsTli>(TopPartitionsByTli1HourName, ESysViewType::ETopPartitionsByTliOneHour);
298316

299317
RegisterPgTablesSystemViews();
300318

301-
RegisterSystemView<Schema::ResourcePoolClassifiers>(ResourcePoolClassifiersName);
302-
RegisterSystemView<Schema::ResourcePools>(ResourcePoolsName);
319+
RegisterSystemView<Schema::ResourcePoolClassifiers>(ResourcePoolClassifiersName, ESysViewType::EResourcePoolClassifiers);
320+
RegisterSystemView<Schema::ResourcePools>(ResourcePoolsName, ESysViewType::EResourcePools);
303321

304322
{
305323
using namespace NAuth;
306-
RegisterSystemView<Schema::AuthUsers>(UsersName);
307-
RegisterSystemView<Schema::AuthGroups>(NAuth::GroupsName);
308-
RegisterSystemView<Schema::AuthGroupMembers>(GroupMembersName);
309-
RegisterSystemView<Schema::AuthOwners>(OwnersName);
310-
RegisterSystemView<Schema::AuthPermissions>(PermissionsName);
311-
RegisterSystemView<Schema::AuthPermissions>(EffectivePermissionsName);
324+
RegisterSystemView<Schema::AuthUsers>(UsersName, ESysViewType::EAuthUsers);
325+
RegisterSystemView<Schema::AuthGroups>(NAuth::GroupsName, ESysViewType::EAuthGroups);
326+
RegisterSystemView<Schema::AuthGroupMembers>(GroupMembersName, ESysViewType::EAuthGroupMembers);
327+
RegisterSystemView<Schema::AuthOwners>(OwnersName, ESysViewType::EAuthOwners);
328+
RegisterSystemView<Schema::AuthPermissions>(PermissionsName, ESysViewType::EAuthPermissions);
329+
RegisterSystemView<Schema::AuthPermissions>(EffectivePermissionsName, ESysViewType::EAuthEffectivePermissions);
312330
}
313331
}
314332

@@ -317,6 +335,7 @@ class TSystemViewResolver : public ISystemViewResolver {
317335
THashMap<TString, TSchema> SubDomainSystemViews;
318336
THashMap<TString, TSchema> OlapStoreSystemViews;
319337
THashMap<TString, TSchema> ColumnTableSystemViews;
338+
THashMap<ESysViewType, TSchema> SystemViews;
320339
};
321340

322341
class TSystemViewRewrittenResolver : public ISystemViewResolver {
@@ -346,6 +365,11 @@ class TSystemViewRewrittenResolver : public ISystemViewResolver {
346365
return view ? TMaybe<TSchema>(*view) : Nothing();
347366
}
348367

368+
TMaybe<TSchema> GetSystemViewSchema(ESysViewType sysViewType) const override final {
369+
Y_UNUSED(sysViewType);
370+
return Nothing();
371+
}
372+
349373
TVector<TString> GetSystemViewNames(ETarget target) const override {
350374
Y_UNUSED(target);
351375
return {};

‎ydb/core/sys_view/common/schema.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
#include "path.h"
44

5+
#include <ydb/core/protos/sys_view_types.pb.h>
56
#include <ydb/core/tablet_flat/flat_cxx_database.h>
67
#include <ydb/core/tx/locks/sys_tables.h>
78
#include <yql/essentials/parser/pg_catalog/catalog.h>
@@ -827,6 +828,8 @@ class ISystemViewResolver {
827828

828829
virtual TMaybe<TSchema> GetSystemViewSchema(const TStringBuf viewName, ETarget target) const = 0;
829830

831+
virtual TMaybe<TSchema> GetSystemViewSchema(NKikimrSysView::ESysViewType viewType) const = 0;
832+
830833
virtual bool IsSystemView(const TStringBuf viewName) const = 0;
831834

832835
virtual TVector<TString> GetSystemViewNames(ETarget target) const = 0;

‎ydb/core/tx/scheme_board/cache.cpp

Lines changed: 46 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,10 @@ namespace {
100100
SetError(context, entry, TResolve::EStatus::LookupError, TKeyDesc::EStatus::NotExists);
101101
}
102102

103+
bool IsVirtualSysView(const TString& cacheItemPath, const TString& sysViewInfo) {
104+
return !cacheItemPath.EndsWith(sysViewInfo);
105+
}
106+
103107
template <typename TEvRequest, typename TDerived>
104108
class TDbResolver: public TActorBootstrapped<TDerived> {
105109
void Handle() {
@@ -240,6 +244,7 @@ namespace {
240244
entry.BlockStoreVolumeInfo.Drop();
241245
entry.FileStoreInfo.Drop();
242246
entry.BackupCollectionInfo.Drop();
247+
entry.SysViewInfo.Drop();
243248
}
244249

245250
static void SetErrorAndClear(TResolveContext* context, TResolve::TEntry& entry, const bool isDescribeDenied) {
@@ -759,6 +764,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
759764
ViewInfo.Drop();
760765
ResourcePoolInfo.Drop();
761766
BackupCollectionInfo.Drop();
767+
SysViewInfo.Drop();
762768
}
763769

764770
void FillTableInfo(const NKikimrSchemeOp::TPathDescription& pathDesc) {
@@ -873,6 +879,18 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
873879
}
874880
}
875881

882+
void FillSystemViewInfo(const NKikimrSchemeOp::TPathDescription& pathDesc) {
883+
if (auto schema = Owner->SystemViewResolver->GetSystemViewSchema(pathDesc.GetSysViewDescription().GetType())) {
884+
Columns = std::move(schema->Columns);
885+
KeyColumnTypes = std::move(schema->KeyColumnTypes);
886+
for (const auto& [id, column] : Columns) {
887+
if (column.IsNotNullColumn) {
888+
NotNullColumns.insert(column.Name);
889+
}
890+
}
891+
}
892+
}
893+
876894
static TResolve::EKind PathSubTypeToTableKind(NKikimrSchemeOp::EPathSubType subType) {
877895
switch (subType) {
878896
case NKikimrSchemeOp::EPathSubTypeSyncIndexImplTable:
@@ -1283,6 +1301,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
12831301
DESCRIPTION_PART(ViewInfo);
12841302
DESCRIPTION_PART(ResourcePoolInfo);
12851303
DESCRIPTION_PART(BackupCollectionInfo);
1304+
DESCRIPTION_PART(SysViewInfo);
12861305

12871306
#undef DESCRIPTION_PART
12881307

@@ -1624,6 +1643,12 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
16241643
FillInfo(Kind, BackupCollectionInfo, std::move(*pathDesc.MutableBackupCollectionDescription()));
16251644
break;
16261645
case NKikimrSchemeOp::EPathTypeSysView:
1646+
Kind = TNavigate::KindSysView;
1647+
if (Created) {
1648+
FillSystemViewInfo(pathDesc);
1649+
}
1650+
FillInfo(Kind, SysViewInfo, std::move(*pathDesc.MutableSysViewDescription()));
1651+
break;
16271652
case NKikimrSchemeOp::EPathTypeInvalid:
16281653
Y_DEBUG_ABORT("Invalid path type");
16291654
break;
@@ -1704,6 +1729,8 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
17041729
ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindBackupCollection);
17051730
break;
17061731
case NKikimrSchemeOp::EPathTypeSysView:
1732+
ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindSysView);
1733+
break;
17071734
case NKikimrSchemeOp::EPathTypeTableIndex:
17081735
case NKikimrSchemeOp::EPathTypeInvalid:
17091736
Y_DEBUG_ABORT("Invalid path type");
@@ -1829,7 +1856,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
18291856
return SetError(context, entry, TNavigate::EStatus::LookupError);
18301857
}
18311858

1832-
if (!entry.TableId.SysViewInfo.empty()) {
1859+
if (!entry.TableId.SysViewInfo.empty() && IsVirtualSysView(Path, entry.TableId.SysViewInfo)) {
18331860
if (Kind == TNavigate::KindPath) {
18341861
auto split = SplitPath(Path);
18351862
if (split.size() == 1) {
@@ -1858,12 +1885,17 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
18581885
|| Kind == TNavigate::KindView;
18591886
const bool isTopic = Kind == TNavigate::KindTopic
18601887
|| Kind == TNavigate::KindCdcStream;
1888+
const bool isSysView = Kind == TNavigate::KindSysView;
18611889

1862-
if (entry.Operation == TNavigate::OpTable && !isTable) {
1890+
if (entry.Operation == TNavigate::OpTable && !(isTable || isSysView)) {
18631891
return SetError(context, entry, TNavigate::EStatus::PathNotTable);
18641892
}
18651893

1866-
if (!Created && (isTable || isTopic)) {
1894+
if (!Created && (isTable || isTopic || isSysView)) {
1895+
return SetError(context, entry, TNavigate::EStatus::PathErrorUnknown);
1896+
}
1897+
1898+
if (isSysView && !Columns) {
18671899
return SetError(context, entry, TNavigate::EStatus::PathErrorUnknown);
18681900
}
18691901

@@ -1888,6 +1920,9 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
18881920
if (entry.RequestType == TNavigate::TEntry::ERequestType::ByPath) {
18891921
if (Kind == TNavigate::KindTable || Kind == TNavigate::KindColumnTable) {
18901922
entry.TableId = TTableId(PathId.OwnerId, PathId.LocalPathId, SchemaVersion);
1923+
} else if (entry.TableId.IsSystemView()) {
1924+
entry.TableId.PathId.OwnerId = PathId.OwnerId;
1925+
entry.TableId.PathId.LocalPathId = PathId.LocalPathId;
18911926
} else {
18921927
entry.TableId = TTableId(PathId.OwnerId, PathId.LocalPathId);
18931928
}
@@ -1927,6 +1962,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
19271962
entry.ViewInfo = ViewInfo;
19281963
entry.ResourcePoolInfo = ResourcePoolInfo;
19291964
entry.BackupCollectionInfo = BackupCollectionInfo;
1965+
entry.SysViewInfo = SysViewInfo;
19301966
}
19311967

19321968
bool CheckColumns(TResolveContext* context, TResolve::TEntry& entry,
@@ -2042,7 +2078,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
20422078
return SetError(context, entry, TResolve::EStatus::LookupError, TKeyDesc::EStatus::NotExists);
20432079
}
20442080

2045-
if (!keyDesc.TableId.SysViewInfo.empty()) {
2081+
if (!keyDesc.TableId.SysViewInfo.empty() && IsVirtualSysView(Path, keyDesc.TableId.SysViewInfo)) {
20462082
if (Kind == TNavigate::KindPath) {
20472083
auto split = SplitPath(Path);
20482084
if (split.size() == 1) {
@@ -2124,7 +2160,8 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
21242160
}
21252161
}
21262162

2127-
if (keyDesc.GetPartitions().empty()) {
2163+
const bool isSysView = Kind == TNavigate::KindSysView;
2164+
if (keyDesc.GetPartitions().empty() && !isSysView) {
21282165
entry.Status = TResolve::EStatus::TypeCheckError;
21292166
keyDesc.Status = TKeyDesc::EStatus::OperationNotSupported;
21302167
}
@@ -2228,6 +2265,10 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
22282265

22292266
// BackupCollection specific
22302267
TIntrusivePtr<TNavigate::TBackupCollectionInfo> BackupCollectionInfo;
2268+
2269+
// SysView specific
2270+
TIntrusivePtr<TNavigate::TSysViewInfo> SysViewInfo;
2271+
22312272
}; // TCacheItem
22322273

22332274
struct TMerger {

0 commit comments

Comments
 (0)