System views creation in SchemeShard under FeatureFlag

Author: n00bcracker

ydb/core/protos/feature_flags.proto

@@ -211,4 +211,5 @@ message TFeatureFlags {
     optional bool EnableNodeBrokerDeltaProtocol = 185 [default = false];
     optional bool EnableAccessToIndexImplTables = 186 [default = false];
     optional bool EnableAddUniqueIndex = 187 [default = false];
+    optional bool EnableMaterializedSystemViewPaths = 188 [default = false, (RequireRestart) = true];
 }

ydb/core/sys_view/common/events.h

@@ -22,7 +22,8 @@ class IDbCounters : public virtual TThrRefBase {
 
 struct TEvSysView {
     enum EEv {
-        EvSendPartitionStats = EventSpaceBegin(TKikimrEvents::ES_SYSTEM_VIEW),
+        EvUpdateFinished = EventSpaceBegin(TKikimrEvents::ES_SYSTEM_VIEW),
+        EvSendPartitionStats,
         EvSetPartitioning,
         EvRemoveTable,
         EvGetPartitionStats,
@@ -81,6 +82,12 @@ struct TEvSysView {
         EvEnd,
     };
 
+    struct TEvUpdateFinished : public TEventLocal<
+        TEvUpdateFinished,
+        EvUpdateFinished>
+    {
+    };
+
     struct TEvSendPartitionStats : public TEventLocal<
         TEvSendPartitionStats,
         EvSendPartitionStats>

ydb/core/sys_view/common/schema.cpp

@@ -210,6 +210,19 @@ class TSystemViewResolver : public ISystemViewResolver {
         return result;
     }
 
+    const THashMap<TString, ESysViewType>& GetSystemViewsTypes(ETarget target) const override {
+        switch (target) {
+        case ETarget::Domain:
+            return DomainSystemViewTypes;
+        case ETarget::SubDomain:
+            return SubDomainSystemViewTypes;
+        case ETarget::OlapStore:
+            return OlapStoreSystemViewTypes;
+        case ETarget::ColumnTable:
+            return  ColumnTableSystemViewTypes;
+        }
+    }
+
     bool IsSystemView(const TStringBuf viewName) const override final {
         return DomainSystemViews.contains(viewName) ||
             SubDomainSystemViews.contains(viewName) ||
@@ -255,13 +268,16 @@ class TSystemViewResolver : public ISystemViewResolver {
     template <typename Table>
     void RegisterSystemView(const TStringBuf& name, ESysViewType type) {
         TSchemaFiller<Table>::Fill(DomainSystemViews[name]);
+        DomainSystemViewTypes[name] = type;
         TSchemaFiller<Table>::Fill(SubDomainSystemViews[name]);
+        SubDomainSystemViewTypes[name] = type;
         TSchemaFiller<Table>::Fill(SystemViews[type]);
     }
 
     template <typename Table>
     void RegisterDomainSystemView(const TStringBuf& name, ESysViewType type) {
         TSchemaFiller<Table>::Fill(DomainSystemViews[name]);
+        DomainSystemViewTypes[name] = type;
         TSchemaFiller<Table>::Fill(SystemViews[type]);
     }
 
@@ -332,9 +348,13 @@ class TSystemViewResolver : public ISystemViewResolver {
 
 private:
     THashMap<TString, TSchema> DomainSystemViews;
+    THashMap<TString, ESysViewType> DomainSystemViewTypes;
     THashMap<TString, TSchema> SubDomainSystemViews;
+    THashMap<TString, ESysViewType> SubDomainSystemViewTypes;
     THashMap<TString, TSchema> OlapStoreSystemViews;
+    THashMap<TString, ESysViewType> OlapStoreSystemViewTypes;
     THashMap<TString, TSchema> ColumnTableSystemViews;
+    THashMap<TString, ESysViewType> ColumnTableSystemViewTypes;
     THashMap<ESysViewType, TSchema> SystemViews;
 };
 
@@ -375,12 +395,18 @@ class TSystemViewRewrittenResolver : public ISystemViewResolver {
         return {};
     }
 
+    const THashMap<TString, ESysViewType>& GetSystemViewsTypes(ETarget target) const override {
+        Y_UNUSED(target);
+        return SystemViewTypes;
+    }
+
     bool IsSystemView(const TStringBuf viewName) const override final {
         return SystemViews.contains(viewName);
     }
 
 private:
     THashMap<TString, TSchema> SystemViews;
+    THashMap<TString, ESysViewType> SystemViewTypes;
 };
 
 ISystemViewResolver* CreateSystemViewResolver() {

ydb/core/sys_view/common/schema.h

@@ -837,6 +837,8 @@ class ISystemViewResolver {
     virtual bool IsSystemView(const TStringBuf viewName) const = 0;
 
     virtual TVector<TString> GetSystemViewNames(ETarget target) const = 0;
+
+    virtual const THashMap<TString, NKikimrSysView::ESysViewType>& GetSystemViewsTypes(ETarget target) const = 0;
 };
 
 ISystemViewResolver* CreateSystemViewResolver();

ydb/core/testlib/basics/feature_flags.h

@@ -77,6 +77,7 @@ class TTestFeatureFlagsHolder {
     FEATURE_FLAG_SETTER(EnableDatabaseAdmin)
     FEATURE_FLAG_SETTER(EnablePermissionsExport)
     FEATURE_FLAG_SETTER(EnableShowCreate)
+    FEATURE_FLAG_SETTER(EnableMaterializedSystemViewPaths)
 
     #undef FEATURE_FLAG_SETTER
 };

ydb/core/tx/schemeshard/schemeshard__operation_create_sysview.cpp

@@ -105,6 +105,7 @@ class TCreateSysView : public TSubOperation {
     THolder<TProposeResponse> Propose(const TString& owner, TOperationContext& context) override {
         const TTabletId ssId = context.SS->SelfTabletId();
 
+        const auto acceptExisting = !Transaction.GetFailOnExist();
         const TString& parentPathStr = Transaction.GetWorkingDir();
         const auto& sysViewDescription = Transaction.GetCreateSysView();
 
@@ -150,7 +151,7 @@ class TCreateSysView : public TSubOperation {
             if (dstPath.IsResolved()) {
                 checks
                     .NotUnderDeleting()
-                    .FailOnExist(TPathElement::EPathType::EPathTypeSysView, /* acceptAlreadyExist */ false);
+                    .FailOnExist(TPathElement::EPathType::EPathTypeSysView, acceptExisting);
             } else {
                 checks
                     .NotEmpty();

ydb/core/tx/schemeshard/schemeshard_impl.cpp

@@ -5,6 +5,8 @@
 #include "olap/bg_tasks/events/global.h"
 #include "schemeshard__data_erasure_manager.h"
 
+#include <ydb/core/sys_view/common/path.h>
+#include <ydb/core/sys_view/common/schema.h>
 #include <ydb/core/tablet_flat/tablet_flat_executed.h>
 #include <ydb/core/tablet/tablet_counters_aggregator.h>
 #include <ydb/core/tablet/tablet_counters_protobuf.h>
@@ -22,6 +24,8 @@
 #include <ydb/core/scheme/scheme_types_proto.h>
 #include <ydb/core/tx/columnshard/bg_tasks/events/events.h>
 #include <ydb/core/tx/scheme_board/events_schemeshard.h>
+#include <ydb/core/tx/schemeshard/schemeshard_path.h>
+#include <ydb/core/tx/schemeshard/schemeshard_sysviews_update.h>
 #include <ydb/core/keyvalue/keyvalue_events.h>
 #include <ydb/library/login/password_checker/password_checker.h>
 #include <ydb/library/login/account_lockout/account_lockout.h>
@@ -71,6 +75,12 @@ bool ResolvePoolNames(
     return true;
 }
 
+struct TDirectoryEntry {
+    NKikimrSchemeOp::EPathType Type;
+    TString Owner;
+    TMaybe<NKikimrSysView::ESysViewType> SysViewType;
+};
+
 }   // anonymous namespace
 
 const TSchemeLimits TSchemeShard::DefaultLimits = {};
@@ -85,6 +95,88 @@ void TSchemeShard::SubscribeToTempTableOwners() {
     }
 }
 
+void TSchemeShard::CollectSysViewUpdates(const TActorContext& ctx) {
+    THashMap<uint64_t ,TModifySysViewRequestInfo> sysViewUpdates;
+
+    const TPath rootPath = TPath::Root(this);
+    const TPath sysViewDirPath = rootPath.Child(TString(NSysView::SysPathName));
+    bool needToMakeSysViewDir = false;
+
+    // make system view dir
+    if (!sysViewDirPath.Check().IsResolved().NotDeleted().NotUnderDeleting()) {
+        needToMakeSysViewDir = true;
+        TModifySysViewRequestInfo makeSysViewDirRequest;
+        makeSysViewDirRequest.OperationType = NKikimrSchemeOp::ESchemeOpMkDir;
+        makeSysViewDirRequest.WorkingDir = rootPath.PathString();
+        makeSysViewDirRequest.TargetName = sysViewDirPath.LeafName();
+
+        sysViewUpdates.emplace(GetCachedTxId(ctx), std::move(makeSysViewDirRequest));
+    }
+
+    const auto sysViewDirType = IsDomainSchemeShard
+        ? NSysView::ISystemViewResolver::ETarget::Domain
+        : NSysView::ISystemViewResolver::ETarget::SubDomain;
+    const auto sysViewsRegister = NSysView::CreateSystemViewResolver()->GetSystemViewsTypes(sysViewDirType);
+
+    TMap<TString, TDirectoryEntry> sysViewDirContents;
+    if (sysViewDirPath.Check().IsResolved().NotDeleted().NotUnderDeleting().IsDirectory()) {
+        for (const auto& [name, pathId] : sysViewDirPath->GetChildren()) {
+            const TPath dirEntryPath = TPath::Init(pathId, this);
+            const auto checks = dirEntryPath.Check();
+            if (checks.IsResolved().NotDeleted().NotUnderDeleting()) {
+                TDirectoryEntry dirEntry;
+                dirEntry.Type = dirEntryPath->PathType;
+                dirEntry.Owner = dirEntryPath->Owner;
+                if (checks.IsSysView()) {
+                    dirEntry.SysViewType = SysViews.at(pathId)->Type;
+                }
+
+                sysViewDirContents.emplace(name, std::move(dirEntry));
+            }
+        }
+    }
+
+    // create absent system views
+    if (needToMakeSysViewDir || sysViewDirPath.Check().IsResolved().NotDeleted().NotUnderDeleting().IsDirectory()) {
+        for (const auto& [name, type] : sysViewsRegister) {
+            if (!sysViewDirContents.contains(name)) {
+                TModifySysViewRequestInfo createSysViewRequest;
+                createSysViewRequest.OperationType = NKikimrSchemeOp::ESchemeOpCreateSysView;
+                createSysViewRequest.WorkingDir = sysViewDirPath.PathString();
+                createSysViewRequest.TargetName = name;
+                createSysViewRequest.SysViewType = type;
+
+                sysViewUpdates.emplace(GetCachedTxId(ctx), std::move(createSysViewRequest));
+            }
+        }
+    }
+
+    THashSet<NKikimrSysView::ESysViewType> availableSysViewTypes;
+    for (const auto& type : std::views::values(sysViewsRegister)) {
+        availableSysViewTypes.insert(type);
+    }
+
+    // drop obsolete system views
+    for (const auto& [name, dirEntry] : sysViewDirContents) {
+        if (dirEntry.Type == NKikimrSchemeOp::EPathTypeSysView) {
+            if (!dirEntry.SysViewType || !availableSysViewTypes.contains(*dirEntry.SysViewType) ||
+                (dirEntry.Owner == BUILTIN_ACL_METADATA && !sysViewsRegister.contains(name))) {
+                TModifySysViewRequestInfo dropSysViewRequest;
+                dropSysViewRequest.OperationType = NKikimrSchemeOp::ESchemeOpDropSysView;
+                dropSysViewRequest.WorkingDir = sysViewDirPath.PathString();
+                dropSysViewRequest.TargetName = name;
+
+                sysViewUpdates.emplace(GetCachedTxId(ctx), std::move(dropSysViewRequest));
+            }
+        }
+    }
+
+    if (!sysViewUpdates.empty()) {
+        LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Start update system views cnt: " << sysViewUpdates.size());
+        Register(CreateSysViewsUpdate(TabletID(), SelfId(), std::move(sysViewUpdates)).Release());
+    }
+}
+
 void TSchemeShard::ActivateAfterInitialization(const TActorContext& ctx, TActivationOpts&& opts) {
     TPathId subDomainPathId = GetCurrentSubDomainPathId();
     TSubDomainInfo::TPtr domainPtr = ResolveDomainInfo(subDomainPathId);
@@ -6680,6 +6772,12 @@ void TSchemeShard::Handle(TEvTxAllocatorClient::TEvAllocateResult::TPtr& ev, con
         for (auto txId: ev->Get()->TxIds) {
             CachedTxIds.push_back(TTxId(txId));
         }
+
+        if (AppData()->FeatureFlags.GetEnableMaterializedSystemViewPaths() && !SysViewsUpdateStarted) {
+            SysViewsUpdateStarted = true;
+            CollectSysViewUpdates(ctx);
+        }
+
         return;
     } else if (Exports.contains(id)) {
         return Execute(CreateTxProgressExport(ev), ctx);

ydb/core/tx/schemeshard/schemeshard_impl.h

@@ -380,6 +380,8 @@ class TSchemeShard
     bool TopicPersistStatsPending = false;
     TStatsQueue<TEvPersQueue::TEvPeriodicTopicStats> TopicStatsQueue;
 
+    bool SysViewsUpdateStarted = false;
+
     TSet<TPathId> CleanDroppedPathsCandidates;
     TSet<TPathId> CleanDroppedSubDomainsCandidates;
     bool CleanDroppedPathsInFly = false;
@@ -915,6 +917,8 @@ class TSchemeShard
 
     void SubscribeToTempTableOwners();
 
+    void CollectSysViewUpdates(const TActorContext& ctx);
+
     void ActivateAfterInitialization(const TActorContext& ctx, TActivationOpts&& opts);
 
     struct TTxInitPopulator;