Skip to content

Commit a4dd474

Browse files
ivanmorozov333ivanmorozov333
authored
Merge 312e3ce into 4b5dfba 
2 parents 4b5dfba + 312e3ce commit a4dd474

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

83 files changed

+1310
-832
lines changed

‎ydb/core/base/events.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,9 @@ struct TKikimrEvents : TEvents {
186186
ES_PRIORITY_QUEUE = 4263,
187187
ES_SOLOMON_PROVIDER = 4264,
188188
ES_CONVEYOR_COMPOSITE = 4265,
189+
ES_GENERAL_CACHE_PUBLIC = 4266,
190+
ES_GENERAL_CACHE_SOURCE = 4267,
191+
189192
};
190193
};
191194

‎ydb/core/driver_lib/run/kikimr_services_initializers.cpp

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,8 @@
192192
#include <ydb/core/tx/conveyor_composite/service/service.h>
193193
#include <ydb/core/tx/conveyor_composite/usage/config.h>
194194
#include <ydb/core/tx/conveyor_composite/usage/service.h>
195+
#include <ydb/core/tx/columnshard/data_accessor/cache_policy/policy.h>
196+
#include <ydb/core/tx/general_cache/usage/service.h>
195197
#include <ydb/core/tx/priorities/usage/config.h>
196198
#include <ydb/core/tx/priorities/usage/service.h>
197199

@@ -2267,6 +2269,29 @@ void TCompPrioritiesInitializer::InitializeServices(NActors::TActorSystemSetup*
22672269
}
22682270
}
22692271

2272+
TGeneralCachePortionsMetadataInitializer::TGeneralCachePortionsMetadataInitializer(const TKikimrRunConfig& runConfig)
2273+
: IKikimrServicesInitializer(runConfig) {
2274+
}
2275+
2276+
void TGeneralCachePortionsMetadataInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) {
2277+
auto serviceConfig = NGeneralCache::NPublic::TConfig::BuildFromProto(Config.GetPortionsMetadataCache());
2278+
if (serviceConfig.IsFail()) {
2279+
AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("error", "cannot parse portions metadata cache config")("action", "default_usage")(
2280+
"error", serviceConfig.GetErrorMessage())("default", NGeneralCache::NPublic::TConfig::BuildDefault().DebugString());
2281+
serviceConfig = NGeneralCache::NPublic::TConfig::BuildDefault();
2282+
}
2283+
AFL_VERIFY(!serviceConfig.IsFail());
2284+
2285+
TIntrusivePtr<::NMonitoring::TDynamicCounters> tabletGroup = GetServiceCounters(appData->Counters, "tablets");
2286+
TIntrusivePtr<::NMonitoring::TDynamicCounters> conveyorGroup = tabletGroup->GetSubgroup("type", "TX_GENERAL_CACHE_PORTIONS_METADATA");
2287+
2288+
auto service = NGeneralCache::TServiceOperator<NOlap::NGeneralCache::TPortionsMetadataCachePolicy>::CreateService(*serviceConfig, conveyorGroup);
2289+
2290+
setup->LocalServices.push_back(
2291+
std::make_pair(NGeneralCache::TServiceOperator<NOlap::NGeneralCache::TPortionsMetadataCachePolicy>::MakeServiceId(NodeId),
2292+
TActorSetupCmd(service, TMailboxType::HTSwap, appData->UserPoolId)));
2293+
}
2294+
22702295
TCompositeConveyorInitializer::TCompositeConveyorInitializer(const TKikimrRunConfig& runConfig)
22712296
: IKikimrServicesInitializer(runConfig) {
22722297
}

‎ydb/core/driver_lib/run/kikimr_services_initializers.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -428,6 +428,12 @@ class TCompositeConveyorInitializer : public IKikimrServicesInitializer {
428428
void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
429429
};
430430

431+
class TGeneralCachePortionsMetadataInitializer: public IKikimrServicesInitializer {
432+
public:
433+
TGeneralCachePortionsMetadataInitializer(const TKikimrRunConfig& runConfig);
434+
void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
435+
};
436+
431437
class TExternalIndexInitializer: public IKikimrServicesInitializer {
432438
public:
433439
TExternalIndexInitializer(const TKikimrRunConfig& runConfig);

‎ydb/core/driver_lib/run/run.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1738,6 +1738,10 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers
17381738
sil->AddServiceInitializer(new TCompositeConveyorInitializer(runConfig));
17391739
}
17401740

1741+
if (serviceMask.EnableGeneralCachePortionsMetadata) {
1742+
sil->AddServiceInitializer(new TGeneralCachePortionsMetadataInitializer(runConfig));
1743+
}
1744+
17411745
if (serviceMask.EnableCms) {
17421746
sil->AddServiceInitializer(new TCmsServiceInitializer(runConfig));
17431747
}

‎ydb/core/driver_lib/run/service_mask.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ union TBasicKikimrServicesMask {
8181
bool EnableGroupedMemoryLimiter:1;
8282
bool EnableAwsService:1;
8383
bool EnableCompPriorities : 1;
84+
bool EnableGeneralCachePortionsMetadata: 1;
8485
};
8586

8687
struct {

‎ydb/core/driver_lib/run/ya.make

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,8 @@ PEERDIR(
106106
ydb/core/tx
107107
ydb/core/tx/columnshard
108108
ydb/core/tx/conveyor/service
109+
ydb/core/tx/general_cache
110+
ydb/core/tx/columnshard/data_accessor/cache_policy
109111
ydb/core/tx/coordinator
110112
ydb/core/tx/datashard
111113
ydb/core/tx/limiter/grouped_memory/usage

‎ydb/core/protos/config.proto

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2337,6 +2337,11 @@ message TLabel {
23372337
optional string Value = 2;
23382338
}
23392339

2340+
message TGeneralCacheConfig {
2341+
optional uint64 MemoryLimit = 1;
2342+
optional uint64 DirectInflightLimit = 2;
2343+
}
2344+
23402345
message TAppConfig {
23412346
option (NMarkers.Root) = true;
23422347
optional TActorSystemConfig ActorSystemConfig = 1;
@@ -2437,6 +2442,7 @@ message TAppConfig {
24372442
optional string StartupConfigYaml = 107;
24382443
optional string StartupStorageYaml = 108;
24392444
optional TCompositeConveyorConfig CompositeConveyorConfig = 109;
2445+
optional TGeneralCacheConfig PortionsMetadataCache = 110;
24402446
}
24412447

24422448
message TYdbVersion {

‎ydb/core/protos/console_config.proto

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ message TConfigItem {
151151
ClusterYamlConfigItem = 101;
152152

153153
CompositeConveyorConfigItem = 109;
154+
PortionsMetadataCacheItem = 110;
154155

155156
// synthetic kinds for audit purposes only
156157
DatabaseYamlConfigChangeItem = 32767;

‎ydb/core/testlib/basics/services.cpp

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include <ydb/core/tablet/node_tablet_monitor.h>
2121
#include <ydb/core/tablet/tablet_list_renderer.h>
2222
#include <ydb/core/tablet_flat/shared_sausagecache.h>
23+
#include <ydb/core/tx/columnshard/data_accessor/cache_policy/policy.h>
2324
#include <ydb/core/tx/scheme_board/replica.h>
2425
#include <ydb/core/client/server/grpc_proxy_status.h>
2526
#include <ydb/core/scheme/tablet_scheme.h>
@@ -166,6 +167,13 @@ namespace NPDisk {
166167
nodeIndex);
167168
}
168169

170+
void SetupCSMetadataCache(TTestActorRuntime& runtime, ui32 nodeIndex) {
171+
auto* actor = NOlap::NDataAccessorControl::TGeneralCache::CreateService(
172+
NGeneralCache::NPublic::TConfig::BuildDefault(), runtime.GetDynamicCounters(nodeIndex));
173+
runtime.AddLocalService(NOlap::NDataAccessorControl::TGeneralCache::MakeServiceId(runtime.GetNodeId(nodeIndex)),
174+
TActorSetupCmd(actor, TMailboxType::ReadAsFilled, 0), nodeIndex);
175+
}
176+
169177
void SetupBlobCache(TTestActorRuntime& runtime, ui32 nodeIndex)
170178
{
171179
runtime.AddLocalService(NBlobCache::MakeBlobCacheServiceId(),
@@ -379,7 +387,8 @@ namespace NPDisk {
379387
SetupTabletPipePerNodeCaches(runtime, nodeIndex, forceFollowers);
380388
SetupResourceBroker(runtime, nodeIndex, app.ResourceBrokerConfig);
381389
SetupSharedPageCache(runtime, nodeIndex, sharedCacheConfig ? *sharedCacheConfig : defaultSharedCacheConfig);
382-
SetupBlobCache(runtime, nodeIndex);
390+
SetupBlobCache(runtime, nodeIndex);
391+
SetupCSMetadataCache(runtime, nodeIndex);
383392
SetupSysViewService(runtime, nodeIndex);
384393
SetupQuoterService(runtime, nodeIndex);
385394
SetupStatService(runtime, nodeIndex);

‎ydb/core/testlib/test_client.cpp

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,8 @@
131131
#include <ydb/core/tx/conveyor_composite/service/service.h>
132132
#include <ydb/core/tx/priorities/usage/service.h>
133133
#include <ydb/core/tx/limiter/grouped_memory/usage/service.h>
134+
#include <ydb/core/tx/columnshard/data_accessor/cache_policy/policy.h>
135+
#include <ydb/core/tx/general_cache/usage/service.h>
134136
#include <ydb/library/folder_service/mock/mock_folder_service_adapter.h>
135137

136138
#include <ydb/core/client/server/ic_nodes_cache_service.h>
@@ -1185,6 +1187,11 @@ namespace Tests {
11851187
const auto aid = Runtime->Register(actor, nodeIdx, appData.UserPoolId, TMailboxType::Revolving, 0);
11861188
Runtime->RegisterService(NConveyorComposite::TServiceOperator::MakeServiceId(Runtime->GetNodeId(nodeIdx)), aid, nodeIdx);
11871189
}
1190+
{
1191+
auto* actor = NOlap::NDataAccessorControl::TGeneralCache::CreateService(NGeneralCache::NPublic::TConfig::BuildDefault(), new ::NMonitoring::TDynamicCounters());
1192+
const auto aid = Runtime->Register(actor, nodeIdx, appData.UserPoolId, TMailboxType::Revolving, 0);
1193+
Runtime->RegisterService(NOlap::NDataAccessorControl::TGeneralCache::MakeServiceId(Runtime->GetNodeId(nodeIdx)), aid, nodeIdx);
1194+
}
11881195
Runtime->Register(CreateLabelsMaintainer({}), nodeIdx, appData.SystemPoolId, TMailboxType::Revolving, 0);
11891196

11901197
auto sysViewService = NSysView::CreateSysViewServiceForTests();

0 commit comments

Comments
 (0)