Skip to content

Commit 6c208f1

Browse files
ivanmorozov333ivanmorozov333
authored
Merge 93ec660 into 2426162 
2 parents 2426162 + 93ec660 commit 6c208f1

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+890
-48
lines changed

‎ydb/core/base/events.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,9 @@ struct TKikimrEvents : TEvents {
186186
ES_PRIORITY_QUEUE = 4263,
187187
ES_SOLOMON_PROVIDER = 4264,
188188
ES_CONVEYOR_COMPOSITE = 4265,
189+
ES_GENERAL_CACHE_PUBLIC = 4266,
190+
ES_GENERAL_CACHE_SOURCE = 4267,
191+
189192
};
190193
};
191194

‎ydb/core/driver_lib/run/kikimr_services_initializers.cpp

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,8 @@
192192
#include <ydb/core/tx/conveyor_composite/service/service.h>
193193
#include <ydb/core/tx/conveyor_composite/usage/config.h>
194194
#include <ydb/core/tx/conveyor_composite/usage/service.h>
195+
#include <ydb/core/tx/columnshard/data_accessor/cache_policy/policy.h>
196+
#include <ydb/core/tx/general_cache/usage/service.h>
195197
#include <ydb/core/tx/priorities/usage/config.h>
196198
#include <ydb/core/tx/priorities/usage/service.h>
197199

@@ -2267,6 +2269,29 @@ void TCompPrioritiesInitializer::InitializeServices(NActors::TActorSystemSetup*
22672269
}
22682270
}
22692271

2272+
TGeneralCachePortionsMetadataInitializer::TGeneralCachePortionsMetadataInitializer(const TKikimrRunConfig& runConfig)
2273+
: IKikimrServicesInitializer(runConfig) {
2274+
}
2275+
2276+
void TGeneralCachePortionsMetadataInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) {
2277+
auto serviceConfig = NGeneralCache::NPublic::TConfig::BuildFromProto(Config.GetPortionsMetadataCache());
2278+
if (serviceConfig.IsFail()) {
2279+
AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("error", "cannot parse portions metadata cache config")("action", "default_usage")(
2280+
"error", serviceConfig.GetErrorMessage())("default", NGeneralCache::NPublic::TConfig::BuildDefault().DebugString());
2281+
serviceConfig = NGeneralCache::NPublic::TConfig::BuildDefault();
2282+
}
2283+
AFL_VERIFY(!serviceConfig.IsFail());
2284+
2285+
TIntrusivePtr<::NMonitoring::TDynamicCounters> tabletGroup = GetServiceCounters(appData->Counters, "tablets");
2286+
TIntrusivePtr<::NMonitoring::TDynamicCounters> conveyorGroup = tabletGroup->GetSubgroup("type", "TX_GENERAL_CACHE_PORTIONS_METADATA");
2287+
2288+
auto service = NGeneralCache::TServiceOperator<NOlap::NGeneralCache::TPortionsMetadataCachePolicy>::CreateService(*serviceConfig, conveyorGroup);
2289+
2290+
setup->LocalServices.push_back(
2291+
std::make_pair(NGeneralCache::TServiceOperator<NOlap::NGeneralCache::TPortionsMetadataCachePolicy>::MakeServiceId(NodeId),
2292+
TActorSetupCmd(service, TMailboxType::HTSwap, appData->UserPoolId)));
2293+
}
2294+
22702295
TCompositeConveyorInitializer::TCompositeConveyorInitializer(const TKikimrRunConfig& runConfig)
22712296
: IKikimrServicesInitializer(runConfig) {
22722297
}

‎ydb/core/driver_lib/run/kikimr_services_initializers.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -428,6 +428,12 @@ class TCompositeConveyorInitializer : public IKikimrServicesInitializer {
428428
void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
429429
};
430430

431+
class TGeneralCachePortionsMetadataInitializer: public IKikimrServicesInitializer {
432+
public:
433+
TGeneralCachePortionsMetadataInitializer(const TKikimrRunConfig& runConfig);
434+
void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
435+
};
436+
431437
class TExternalIndexInitializer: public IKikimrServicesInitializer {
432438
public:
433439
TExternalIndexInitializer(const TKikimrRunConfig& runConfig);

‎ydb/core/driver_lib/run/run.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1738,6 +1738,10 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers
17381738
sil->AddServiceInitializer(new TCompositeConveyorInitializer(runConfig));
17391739
}
17401740

1741+
if (serviceMask.EnableGeneralCachePortionsMetadata) {
1742+
sil->AddServiceInitializer(new TGeneralCachePortionsMetadataInitializer(runConfig));
1743+
}
1744+
17411745
if (serviceMask.EnableCms) {
17421746
sil->AddServiceInitializer(new TCmsServiceInitializer(runConfig));
17431747
}

‎ydb/core/driver_lib/run/service_mask.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ union TBasicKikimrServicesMask {
8181
bool EnableGroupedMemoryLimiter:1;
8282
bool EnableAwsService:1;
8383
bool EnableCompPriorities : 1;
84+
bool EnableGeneralCachePortionsMetadata: 1;
8485
};
8586

8687
struct {

‎ydb/core/driver_lib/run/ya.make

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,8 @@ PEERDIR(
106106
ydb/core/tx
107107
ydb/core/tx/columnshard
108108
ydb/core/tx/conveyor/service
109+
ydb/core/tx/general_cache
110+
ydb/core/tx/columnshard/data_accessor/cache_policy
109111
ydb/core/tx/coordinator
110112
ydb/core/tx/datashard
111113
ydb/core/tx/limiter/grouped_memory/usage

‎ydb/core/protos/config.proto

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2337,6 +2337,11 @@ message TLabel {
23372337
optional string Value = 2;
23382338
}
23392339

2340+
message TGeneralCacheConfig {
2341+
optional uint64 MemoryLimit = 1;
2342+
optional uint64 DirectInflightLimit = 2;
2343+
}
2344+
23402345
message TAppConfig {
23412346
option (NMarkers.Root) = true;
23422347
optional TActorSystemConfig ActorSystemConfig = 1;
@@ -2437,6 +2442,7 @@ message TAppConfig {
24372442
optional string StartupConfigYaml = 107;
24382443
optional string StartupStorageYaml = 108;
24392444
optional TCompositeConveyorConfig CompositeConveyorConfig = 109;
2445+
optional TGeneralCacheConfig PortionsMetadataCache = 110;
24402446
}
24412447

24422448
message TYdbVersion {

‎ydb/core/protos/console_config.proto

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ message TConfigItem {
151151
ClusterYamlConfigItem = 101;
152152

153153
CompositeConveyorConfigItem = 109;
154+
PortionsMetadataCacheItem = 110;
154155

155156
// synthetic kinds for audit purposes only
156157
DatabaseYamlConfigChangeItem = 32767;

‎ydb/core/tx/columnshard/blobs_action/counters/storage.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ enum class EConsumer {
3030
WRITING_OPERATOR,
3131
NORMALIZER,
3232
STATISTICS,
33+
STATISTICS_FLUSH,
34+
FETCH_ON_LOAD,
3335
UNDEFINED,
3436

3537
COUNT

‎ydb/core/tx/columnshard/columnshard__statistics.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#include "columnshard.h"
22
#include "columnshard_impl.h"
33

4+
#include "data_accessor/cache_policy/policy.h"
45
#include "ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/meta.h"
56

67
#include <ydb/core/protos/kqp.pb.h>
@@ -253,7 +254,8 @@ class TColumnPortionsAccumulator {
253254
return;
254255
}
255256
Result->AddWaitingTask();
256-
std::shared_ptr<NOlap::TDataAccessorsRequest> request = std::make_shared<NOlap::TDataAccessorsRequest>("STATISTICS_FLUSH");
257+
std::shared_ptr<NOlap::TDataAccessorsRequest> request =
258+
std::make_shared<NOlap::TDataAccessorsRequest>(NOlap::NGeneralCache::TPortionsMetadataCachePolicy::EConsumer::STATISTICS_FLUSH);
257259
for (auto&& i : Portions) {
258260
request->AddPortion(i);
259261
}
@@ -304,7 +306,7 @@ void TColumnShard::Handle(NStat::TEvStatistics::TEvStatisticsRequest::TPtr& ev,
304306
columnTagsRequested = std::set<ui32>(allColumnIds.begin(), allColumnIds.end());
305307
}
306308

307-
NOlap::TDataAccessorsRequest request("STATISTICS");
309+
NOlap::TDataAccessorsRequest request(NOlap::NGeneralCache::TPortionsMetadataCachePolicy::EConsumer::STATISTICS);
308310
std::shared_ptr<TResultAccumulator> resultAccumulator =
309311
std::make_shared<TResultAccumulator>(columnTagsRequested, ev->Sender, ev->Cookie, std::move(response));
310312
auto versionedIndex = std::make_shared<NOlap::TVersionedIndex>(index.GetVersionedIndex());

0 commit comments

Comments
 (0)