Skip to content

Commit 33b7f67

Browse files
ivanmorozov333ivanmorozov333
andauthored
portion producing method improve (#19200)
Co-authored-by: ivanmorozov333 <[email protected]>
1 parent 9051749 commit 33b7f67

File tree

26 files changed

+51
-92
lines changed

26 files changed

+51
-92
lines changed

ydb/core/tx/columnshard/counters/portion_index.cpp

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,7 @@
55
namespace NKikimr::NColumnShard {
66

77
TPortionIndexStats::TPortionClass::TPortionClass(const NOlap::TPortionInfo& portion) {
8-
if (portion.HasRemoveSnapshot()) {
9-
Produced = NOlap::NPortion::EProduced::INACTIVE;
10-
} else if (portion.GetTierNameDef(NOlap::NBlobOperations::TGlobal::DefaultStorageId) != NOlap::NBlobOperations::TGlobal::DefaultStorageId) {
11-
Produced = NOlap::NPortion::EProduced::EVICTED;
12-
} else {
13-
Produced = portion.GetMeta().GetProduced();
14-
}
8+
Produced = portion.GetProduced();
159
}
1610

1711
void TPortionIndexStats::AddPortion(const NOlap::TPortionInfo& portion) {

ydb/core/tx/columnshard/engines/changes/compaction.cpp

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,8 @@ void TCompactColumnEngineChanges::DoDebugString(TStringOutput& out) const {
2323
void TCompactColumnEngineChanges::DoCompile(TFinalizationContext& context) {
2424
TBase::DoCompile(context);
2525

26-
const TPortionMeta::EProduced producedClassResultCompaction = GetResultProducedClass();
2726
for (auto& portionInfo : AppendedPortions) {
2827
auto& constructor = portionInfo.GetPortionConstructor().MutablePortionConstructor();
29-
constructor.MutableMeta().UpdateRecordsMeta(producedClassResultCompaction);
3028
constructor.MutableMeta().SetCompactionLevel(GranuleMeta->GetOptimizerPlanner().GetAppropriateLevel(
3129
GetPortionsToMove().GetTargetCompactionLevel().value_or(0), portionInfo.GetPortionConstructor()));
3230
}

ydb/core/tx/columnshard/engines/changes/compaction.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ class TCompactColumnEngineChanges: public TChangesWithAppend {
2222
virtual void DoOnFinish(NColumnShard::TColumnShard& self, TChangesFinishContext& context) override;
2323
virtual void DoDebugString(TStringOutput& out) const override;
2424
virtual void DoCompile(TFinalizationContext& context) override;
25-
virtual TPortionMeta::EProduced GetResultProducedClass() const = 0;
25+
virtual NPortion::EProduced GetResultProducedClass() const = 0;
2626
virtual void OnAbortEmergency() override {
2727
NeedGranuleStatusProvide = false;
2828
}

ydb/core/tx/columnshard/engines/changes/general_compaction.cpp

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,7 @@ std::vector<TWritePortionInfoWithBlobsResult> TGeneralCompactColumnEngineChanges
2929
if (shardingActual) {
3030
shardingActualVersion = shardingActual->GetSnapshotVersion();
3131
}
32-
auto result = merger.Execute(stats, CheckPoints, resultFiltered, GranuleMeta->GetPathId(), shardingActualVersion);
33-
for (auto&& p : result) {
34-
p.GetPortionConstructor().MutablePortionConstructor().MutableMeta().UpdateRecordsMeta(NPortion::EProduced::SPLIT_COMPACTED);
35-
}
36-
return result;
32+
return merger.Execute(stats, CheckPoints, resultFiltered, GranuleMeta->GetPathId(), shardingActualVersion);
3733
}
3834

3935
TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstructionContext& context) noexcept {
@@ -42,12 +38,12 @@ TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstruc
4238
THashMap<ui32, TSimplePortionsGroupInfo> portionGroups;
4339
for (auto&& i : SwitchedPortions) {
4440
portionGroups[i->GetMeta().GetCompactionLevel()].AddPortion(i);
45-
if (i->GetMeta().GetProduced() == TPortionMeta::EProduced::INSERTED) {
41+
if (i->GetProduced() == NPortion::EProduced::INSERTED) {
4642
insertedPortions.AddPortion(i);
47-
} else if (i->GetMeta().GetProduced() == TPortionMeta::EProduced::SPLIT_COMPACTED) {
43+
} else if (i->GetProduced() == NPortion::EProduced::SPLIT_COMPACTED) {
4844
compactedPortions.AddPortion(i);
4945
} else {
50-
AFL_VERIFY(false);
46+
AFL_VERIFY(false)("portion_prod", i->GetProduced())("portion_type", i->GetPortionType());
5147
}
5248
}
5349
NChanges::TGeneralCompactionCounters::OnRepackPortions(insertedPortions + compactedPortions);

ydb/core/tx/columnshard/engines/changes/general_compaction.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@ class TGeneralCompactColumnEngineChanges: public TCompactColumnEngineChanges,
3333
return true;
3434
}
3535

36-
virtual TPortionMeta::EProduced GetResultProducedClass() const override {
37-
return TPortionMeta::EProduced::SPLIT_COMPACTED;
36+
virtual NPortion::EProduced GetResultProducedClass() const override {
37+
return NPortion::EProduced::SPLIT_COMPACTED;
3838
}
3939
virtual void DoStart(NColumnShard::TColumnShard& self) override;
4040
virtual NColumnShard::ECumulativeCounters GetCounterIndex(const bool isSuccess) const override;

ydb/core/tx/columnshard/engines/changes/indexation.cpp

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,6 @@ TConclusionStatus TInsertColumnEngineChanges::DoConstructBlobs(TConstructionCont
285285
merger.SetOptimizationWritingPackMode(true);
286286
auto localAppended = merger.Execute(stats, itGranule->second, filteredSnapshot, pathId, shardingVersion);
287287
for (auto&& i : localAppended) {
288-
i.GetPortionConstructor().MutablePortionConstructor().MutableMeta().UpdateRecordsMeta(NPortion::EProduced::INSERTED);
289288
AppendedPortions.emplace_back(std::move(i));
290289
}
291290
}

ydb/core/tx/columnshard/engines/changes/with_appended.cpp

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -42,22 +42,22 @@ void TChangesWithAppend::DoWriteIndexOnComplete(NColumnShard::TColumnShard* self
4242
for (auto& portionBuilder : AppendedPortions) {
4343
auto& portionInfo = portionBuilder.GetPortionResult();
4444
sb << portionInfo.GetPortionInfo().GetPortionId() << ",";
45-
switch (portionInfo.GetPortionInfo().GetMeta().Produced) {
46-
case NOlap::TPortionMeta::EProduced::UNSPECIFIED:
45+
switch (portionInfo.GetPortionInfo().GetProduced()) {
46+
case NOlap::NPortion::EProduced::UNSPECIFIED:
4747
Y_ABORT_UNLESS(false); // unexpected
48-
case NOlap::TPortionMeta::EProduced::INSERTED:
48+
case NOlap::NPortion::EProduced::INSERTED:
4949
self->Counters.GetTabletCounters()->IncCounter(NColumnShard::COUNTER_INDEXING_PORTIONS_WRITTEN);
5050
break;
51-
case NOlap::TPortionMeta::EProduced::COMPACTED:
51+
case NOlap::NPortion::EProduced::COMPACTED:
5252
self->Counters.GetTabletCounters()->IncCounter(NColumnShard::COUNTER_COMPACTION_PORTIONS_WRITTEN);
5353
break;
54-
case NOlap::TPortionMeta::EProduced::SPLIT_COMPACTED:
54+
case NOlap::NPortion::EProduced::SPLIT_COMPACTED:
5555
self->Counters.GetTabletCounters()->IncCounter(NColumnShard::COUNTER_SPLIT_COMPACTION_PORTIONS_WRITTEN);
5656
break;
57-
case NOlap::TPortionMeta::EProduced::EVICTED:
58-
Y_ABORT("Unexpected evicted case");
57+
case NOlap::NPortion::EProduced::EVICTED:
58+
self->Counters.GetTabletCounters()->IncCounter(NColumnShard::COUNTER_EVICTION_PORTIONS_WRITTEN);
5959
break;
60-
case NOlap::TPortionMeta::EProduced::INACTIVE:
60+
case NOlap::NPortion::EProduced::INACTIVE:
6161
Y_ABORT("Unexpected inactive case");
6262
break;
6363
}

ydb/core/tx/columnshard/engines/db_wrapper.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,9 @@ void TDbWrapper::WriteColumn(const NOlap::TPortionInfo& portion, const TColumnRe
6060
}
6161
if (AppDataVerified().ColumnShardConfig.GetColumnChunksV0Usage()) {
6262
if (row.GetChunkIdx() == 0 && row.GetColumnId() == firstPKColumnId) {
63-
*rowProto.MutablePortionMeta() = portion.GetMeta().SerializeToProto();
63+
*rowProto.MutablePortionMeta() =
64+
portion.GetMeta().SerializeToProto(portion.GetPortionType() == EPortionType::Compacted ? NPortion::EProduced::SPLIT_COMPACTED
65+
: NPortion::EProduced::INSERTED);
6466
}
6567
using IndexColumns = NColumnShard::Schema::IndexColumns;
6668
auto removeSnapshot = portion.GetRemoveSnapshotOptional();

ydb/core/tx/columnshard/engines/portions/compacted.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
namespace NKikimr::NOlap {
88

99
void TCompactedPortionInfo::DoSaveMetaToDatabase(NIceDb::TNiceDb& db) const {
10-
auto metaProto = GetMeta().SerializeToProto();
10+
auto metaProto = GetMeta().SerializeToProto(NPortion::EProduced::SPLIT_COMPACTED);
1111
using IndexPortions = NColumnShard::Schema::IndexPortions;
1212
const auto removeSnapshot = GetRemoveSnapshotOptional();
1313
db.Table<IndexPortions>()

ydb/core/tx/columnshard/engines/portions/constructor_meta.cpp

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,6 @@ TPortionMetaConstructor::TPortionMetaConstructor(const TPortionMeta& meta, const
3939
if (withBlobs) {
4040
BlobIds = meta.BlobIds;
4141
}
42-
if (meta.Produced != NPortion::EProduced::UNSPECIFIED) {
43-
Produced = meta.Produced;
44-
}
4542
}
4643

4744
TPortionMeta TPortionMetaConstructor::Build() {
@@ -65,7 +62,6 @@ TPortionMeta TPortionMetaConstructor::Build() {
6562
result.BlobIds.shrink_to_fit();
6663
result.CompactionLevel = *TValidator::CheckNotNull(CompactionLevel);
6764
result.DeletionsCount = *TValidator::CheckNotNull(DeletionsCount);
68-
result.Produced = *TValidator::CheckNotNull(Produced);
6965

7066
result.RecordsCount = *TValidator::CheckNotNull(RecordsCount);
7167
result.ColumnRawBytes = *TValidator::CheckNotNull(ColumnRawBytes);
@@ -80,7 +76,6 @@ TPortionMeta TPortionMetaConstructor::Build() {
8076

8177
bool TPortionMetaConstructor::LoadMetadata(
8278
const NKikimrTxColumnShard::TIndexPortionMeta& portionMeta, const TIndexInfo& indexInfo, const IBlobGroupSelector& groupSelector) {
83-
AFL_VERIFY(!Produced)("produced", Produced);
8479
if (portionMeta.GetTierName()) {
8580
TierName = portionMeta.GetTierName();
8681
}
@@ -99,20 +94,6 @@ bool TPortionMetaConstructor::LoadMetadata(
9994
ColumnBlobBytes = TValidator::CheckNotNull(portionMeta.GetColumnBlobBytes());
10095
IndexRawBytes = portionMeta.GetIndexRawBytes();
10196
IndexBlobBytes = portionMeta.GetIndexBlobBytes();
102-
if (portionMeta.GetIsInserted()) {
103-
Produced = TPortionMeta::EProduced::INSERTED;
104-
} else if (portionMeta.GetIsCompacted()) {
105-
Produced = TPortionMeta::EProduced::COMPACTED;
106-
} else if (portionMeta.GetIsSplitCompacted()) {
107-
Produced = TPortionMeta::EProduced::SPLIT_COMPACTED;
108-
} else if (portionMeta.GetIsEvicted()) {
109-
Produced = TPortionMeta::EProduced::EVICTED;
110-
} else {
111-
AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("event", "DeserializeFromProto")("error", "incorrect portion meta")(
112-
"meta", portionMeta.DebugString());
113-
return false;
114-
}
115-
AFL_VERIFY(Produced != TPortionMeta::EProduced::UNSPECIFIED);
11697
if (portionMeta.HasPrimaryKeyBordersV1()) {
11798
FirstAndLastPK = NArrow::TFirstLastSpecialKeys(
11899
portionMeta.GetPrimaryKeyBordersV1().GetFirst(), portionMeta.GetPrimaryKeyBordersV1().GetLast(), indexInfo.GetReplaceKey());

0 commit comments

Comments
 (0)