diff --git a/ydb/core/tx/columnshard/counters/portion_index.cpp b/ydb/core/tx/columnshard/counters/portion_index.cpp index 5e3c8ff7ce28..60fd40b5fa6a 100644 --- a/ydb/core/tx/columnshard/counters/portion_index.cpp +++ b/ydb/core/tx/columnshard/counters/portion_index.cpp @@ -5,13 +5,7 @@ namespace NKikimr::NColumnShard { TPortionIndexStats::TPortionClass::TPortionClass(const NOlap::TPortionInfo& portion) { - if (portion.HasRemoveSnapshot()) { - Produced = NOlap::NPortion::EProduced::INACTIVE; - } else if (portion.GetTierNameDef(NOlap::NBlobOperations::TGlobal::DefaultStorageId) != NOlap::NBlobOperations::TGlobal::DefaultStorageId) { - Produced = NOlap::NPortion::EProduced::EVICTED; - } else { - Produced = portion.GetMeta().GetProduced(); - } + Produced = portion.GetProduced(); } void TPortionIndexStats::AddPortion(const NOlap::TPortionInfo& portion) { diff --git a/ydb/core/tx/columnshard/engines/changes/compaction.cpp b/ydb/core/tx/columnshard/engines/changes/compaction.cpp index b86c5ff28edd..01b593d47793 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction.cpp @@ -23,10 +23,8 @@ void TCompactColumnEngineChanges::DoDebugString(TStringOutput& out) const { void TCompactColumnEngineChanges::DoCompile(TFinalizationContext& context) { TBase::DoCompile(context); - const TPortionMeta::EProduced producedClassResultCompaction = GetResultProducedClass(); for (auto& portionInfo : AppendedPortions) { auto& constructor = portionInfo.GetPortionConstructor().MutablePortionConstructor(); - constructor.MutableMeta().UpdateRecordsMeta(producedClassResultCompaction); constructor.MutableMeta().SetCompactionLevel(GranuleMeta->GetOptimizerPlanner().GetAppropriateLevel( GetPortionsToMove().GetTargetCompactionLevel().value_or(0), portionInfo.GetPortionConstructor())); } diff --git a/ydb/core/tx/columnshard/engines/changes/compaction.h b/ydb/core/tx/columnshard/engines/changes/compaction.h index 1c12ae942c6b..1178b27eb97f 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction.h +++ b/ydb/core/tx/columnshard/engines/changes/compaction.h @@ -22,7 +22,7 @@ class TCompactColumnEngineChanges: public TChangesWithAppend { virtual void DoOnFinish(NColumnShard::TColumnShard& self, TChangesFinishContext& context) override; virtual void DoDebugString(TStringOutput& out) const override; virtual void DoCompile(TFinalizationContext& context) override; - virtual TPortionMeta::EProduced GetResultProducedClass() const = 0; + virtual NPortion::EProduced GetResultProducedClass() const = 0; virtual void OnAbortEmergency() override { NeedGranuleStatusProvide = false; } diff --git a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp index f40719145ea3..665928fa0b79 100644 --- a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp +++ b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp @@ -29,11 +29,7 @@ std::vector TGeneralCompactColumnEngineChanges if (shardingActual) { shardingActualVersion = shardingActual->GetSnapshotVersion(); } - auto result = merger.Execute(stats, CheckPoints, resultFiltered, GranuleMeta->GetPathId(), shardingActualVersion); - for (auto&& p : result) { - p.GetPortionConstructor().MutablePortionConstructor().MutableMeta().UpdateRecordsMeta(NPortion::EProduced::SPLIT_COMPACTED); - } - return result; + return merger.Execute(stats, CheckPoints, resultFiltered, GranuleMeta->GetPathId(), shardingActualVersion); } TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstructionContext& context) noexcept { @@ -42,12 +38,12 @@ TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstruc THashMap portionGroups; for (auto&& i : SwitchedPortions) { portionGroups[i->GetMeta().GetCompactionLevel()].AddPortion(i); - if (i->GetMeta().GetProduced() == TPortionMeta::EProduced::INSERTED) { + if (i->GetProduced() == NPortion::EProduced::INSERTED) { insertedPortions.AddPortion(i); - } else if (i->GetMeta().GetProduced() == TPortionMeta::EProduced::SPLIT_COMPACTED) { + } else if (i->GetProduced() == NPortion::EProduced::SPLIT_COMPACTED) { compactedPortions.AddPortion(i); } else { - AFL_VERIFY(false); + AFL_VERIFY(false)("portion_prod", i->GetProduced())("portion_type", i->GetPortionType()); } } NChanges::TGeneralCompactionCounters::OnRepackPortions(insertedPortions + compactedPortions); diff --git a/ydb/core/tx/columnshard/engines/changes/general_compaction.h b/ydb/core/tx/columnshard/engines/changes/general_compaction.h index 9fc042b6dc6e..6d4ab460077b 100644 --- a/ydb/core/tx/columnshard/engines/changes/general_compaction.h +++ b/ydb/core/tx/columnshard/engines/changes/general_compaction.h @@ -33,8 +33,8 @@ class TGeneralCompactColumnEngineChanges: public TCompactColumnEngineChanges, return true; } - virtual TPortionMeta::EProduced GetResultProducedClass() const override { - return TPortionMeta::EProduced::SPLIT_COMPACTED; + virtual NPortion::EProduced GetResultProducedClass() const override { + return NPortion::EProduced::SPLIT_COMPACTED; } virtual void DoStart(NColumnShard::TColumnShard& self) override; virtual NColumnShard::ECumulativeCounters GetCounterIndex(const bool isSuccess) const override; diff --git a/ydb/core/tx/columnshard/engines/changes/indexation.cpp b/ydb/core/tx/columnshard/engines/changes/indexation.cpp index e464d6c46e92..dc1f8bd00edc 100644 --- a/ydb/core/tx/columnshard/engines/changes/indexation.cpp +++ b/ydb/core/tx/columnshard/engines/changes/indexation.cpp @@ -285,7 +285,6 @@ TConclusionStatus TInsertColumnEngineChanges::DoConstructBlobs(TConstructionCont merger.SetOptimizationWritingPackMode(true); auto localAppended = merger.Execute(stats, itGranule->second, filteredSnapshot, pathId, shardingVersion); for (auto&& i : localAppended) { - i.GetPortionConstructor().MutablePortionConstructor().MutableMeta().UpdateRecordsMeta(NPortion::EProduced::INSERTED); AppendedPortions.emplace_back(std::move(i)); } } diff --git a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp index 7006793a71cd..c69bd0176598 100644 --- a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp +++ b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp @@ -42,22 +42,22 @@ void TChangesWithAppend::DoWriteIndexOnComplete(NColumnShard::TColumnShard* self for (auto& portionBuilder : AppendedPortions) { auto& portionInfo = portionBuilder.GetPortionResult(); sb << portionInfo.GetPortionInfo().GetPortionId() << ","; - switch (portionInfo.GetPortionInfo().GetMeta().Produced) { - case NOlap::TPortionMeta::EProduced::UNSPECIFIED: + switch (portionInfo.GetPortionInfo().GetProduced()) { + case NOlap::NPortion::EProduced::UNSPECIFIED: Y_ABORT_UNLESS(false); // unexpected - case NOlap::TPortionMeta::EProduced::INSERTED: + case NOlap::NPortion::EProduced::INSERTED: self->Counters.GetTabletCounters()->IncCounter(NColumnShard::COUNTER_INDEXING_PORTIONS_WRITTEN); break; - case NOlap::TPortionMeta::EProduced::COMPACTED: + case NOlap::NPortion::EProduced::COMPACTED: self->Counters.GetTabletCounters()->IncCounter(NColumnShard::COUNTER_COMPACTION_PORTIONS_WRITTEN); break; - case NOlap::TPortionMeta::EProduced::SPLIT_COMPACTED: + case NOlap::NPortion::EProduced::SPLIT_COMPACTED: self->Counters.GetTabletCounters()->IncCounter(NColumnShard::COUNTER_SPLIT_COMPACTION_PORTIONS_WRITTEN); break; - case NOlap::TPortionMeta::EProduced::EVICTED: - Y_ABORT("Unexpected evicted case"); + case NOlap::NPortion::EProduced::EVICTED: + self->Counters.GetTabletCounters()->IncCounter(NColumnShard::COUNTER_EVICTION_PORTIONS_WRITTEN); break; - case NOlap::TPortionMeta::EProduced::INACTIVE: + case NOlap::NPortion::EProduced::INACTIVE: Y_ABORT("Unexpected inactive case"); break; } diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.cpp b/ydb/core/tx/columnshard/engines/db_wrapper.cpp index 768356e63ae7..584fe3417cdf 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.cpp +++ b/ydb/core/tx/columnshard/engines/db_wrapper.cpp @@ -60,7 +60,9 @@ void TDbWrapper::WriteColumn(const NOlap::TPortionInfo& portion, const TColumnRe } if (AppDataVerified().ColumnShardConfig.GetColumnChunksV0Usage()) { if (row.GetChunkIdx() == 0 && row.GetColumnId() == firstPKColumnId) { - *rowProto.MutablePortionMeta() = portion.GetMeta().SerializeToProto(); + *rowProto.MutablePortionMeta() = + portion.GetMeta().SerializeToProto(portion.GetPortionType() == EPortionType::Compacted ? NPortion::EProduced::SPLIT_COMPACTED + : NPortion::EProduced::INSERTED); } using IndexColumns = NColumnShard::Schema::IndexColumns; auto removeSnapshot = portion.GetRemoveSnapshotOptional(); diff --git a/ydb/core/tx/columnshard/engines/portions/compacted.cpp b/ydb/core/tx/columnshard/engines/portions/compacted.cpp index 1ec51382fbd1..3464784009c2 100644 --- a/ydb/core/tx/columnshard/engines/portions/compacted.cpp +++ b/ydb/core/tx/columnshard/engines/portions/compacted.cpp @@ -7,7 +7,7 @@ namespace NKikimr::NOlap { void TCompactedPortionInfo::DoSaveMetaToDatabase(NIceDb::TNiceDb& db) const { - auto metaProto = GetMeta().SerializeToProto(); + auto metaProto = GetMeta().SerializeToProto(NPortion::EProduced::SPLIT_COMPACTED); using IndexPortions = NColumnShard::Schema::IndexPortions; const auto removeSnapshot = GetRemoveSnapshotOptional(); db.Table() diff --git a/ydb/core/tx/columnshard/engines/portions/constructor_meta.cpp b/ydb/core/tx/columnshard/engines/portions/constructor_meta.cpp index 77ea5eb91ff4..9da7c73f17c0 100644 --- a/ydb/core/tx/columnshard/engines/portions/constructor_meta.cpp +++ b/ydb/core/tx/columnshard/engines/portions/constructor_meta.cpp @@ -39,9 +39,6 @@ TPortionMetaConstructor::TPortionMetaConstructor(const TPortionMeta& meta, const if (withBlobs) { BlobIds = meta.BlobIds; } - if (meta.Produced != NPortion::EProduced::UNSPECIFIED) { - Produced = meta.Produced; - } } TPortionMeta TPortionMetaConstructor::Build() { @@ -65,7 +62,6 @@ TPortionMeta TPortionMetaConstructor::Build() { result.BlobIds.shrink_to_fit(); result.CompactionLevel = *TValidator::CheckNotNull(CompactionLevel); result.DeletionsCount = *TValidator::CheckNotNull(DeletionsCount); - result.Produced = *TValidator::CheckNotNull(Produced); result.RecordsCount = *TValidator::CheckNotNull(RecordsCount); result.ColumnRawBytes = *TValidator::CheckNotNull(ColumnRawBytes); @@ -80,7 +76,6 @@ TPortionMeta TPortionMetaConstructor::Build() { bool TPortionMetaConstructor::LoadMetadata( const NKikimrTxColumnShard::TIndexPortionMeta& portionMeta, const TIndexInfo& indexInfo, const IBlobGroupSelector& groupSelector) { - AFL_VERIFY(!Produced)("produced", Produced); if (portionMeta.GetTierName()) { TierName = portionMeta.GetTierName(); } @@ -99,20 +94,6 @@ bool TPortionMetaConstructor::LoadMetadata( ColumnBlobBytes = TValidator::CheckNotNull(portionMeta.GetColumnBlobBytes()); IndexRawBytes = portionMeta.GetIndexRawBytes(); IndexBlobBytes = portionMeta.GetIndexBlobBytes(); - if (portionMeta.GetIsInserted()) { - Produced = TPortionMeta::EProduced::INSERTED; - } else if (portionMeta.GetIsCompacted()) { - Produced = TPortionMeta::EProduced::COMPACTED; - } else if (portionMeta.GetIsSplitCompacted()) { - Produced = TPortionMeta::EProduced::SPLIT_COMPACTED; - } else if (portionMeta.GetIsEvicted()) { - Produced = TPortionMeta::EProduced::EVICTED; - } else { - AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("event", "DeserializeFromProto")("error", "incorrect portion meta")( - "meta", portionMeta.DebugString()); - return false; - } - AFL_VERIFY(Produced != TPortionMeta::EProduced::UNSPECIFIED); if (portionMeta.HasPrimaryKeyBordersV1()) { FirstAndLastPK = NArrow::TFirstLastSpecialKeys( portionMeta.GetPrimaryKeyBordersV1().GetFirst(), portionMeta.GetPrimaryKeyBordersV1().GetLast(), indexInfo.GetReplaceKey()); diff --git a/ydb/core/tx/columnshard/engines/portions/constructor_meta.h b/ydb/core/tx/columnshard/engines/portions/constructor_meta.h index fe7de12fe30e..a179abdc353e 100644 --- a/ydb/core/tx/columnshard/engines/portions/constructor_meta.h +++ b/ydb/core/tx/columnshard/engines/portions/constructor_meta.h @@ -17,7 +17,6 @@ class TPortionMetaConstructor: public TPortionMetaBase { std::optional TierName; std::optional RecordSnapshotMin; std::optional RecordSnapshotMax; - std::optional Produced; std::optional CompactionLevel; std::optional RecordsCount; @@ -70,10 +69,6 @@ class TPortionMetaConstructor: public TPortionMetaBase { SetTierName(tierName); } - void UpdateRecordsMeta(const NPortion::EProduced prod) { - Produced = prod; - } - TPortionMeta Build(); [[nodiscard]] bool LoadMetadata( diff --git a/ydb/core/tx/columnshard/engines/portions/constructor_portion.cpp b/ydb/core/tx/columnshard/engines/portions/constructor_portion.cpp index f6e17be86010..f345d93e6cc1 100644 --- a/ydb/core/tx/columnshard/engines/portions/constructor_portion.cpp +++ b/ydb/core/tx/columnshard/engines/portions/constructor_portion.cpp @@ -56,8 +56,6 @@ std::shared_ptr TWrittenPortionInfoConstructor::BuildPortionImpl(T } AFL_VERIFY(InsertWriteId); result->InsertWriteId = *InsertWriteId; - - AFL_VERIFY(result->GetMeta().GetProduced() == NPortion::EProduced::INSERTED); return result; } diff --git a/ydb/core/tx/columnshard/engines/portions/meta.cpp b/ydb/core/tx/columnshard/engines/portions/meta.cpp index d4655b5739cc..b0811125f974 100644 --- a/ydb/core/tx/columnshard/engines/portions/meta.cpp +++ b/ydb/core/tx/columnshard/engines/portions/meta.cpp @@ -10,7 +10,7 @@ namespace NKikimr::NOlap { -NKikimrTxColumnShard::TIndexPortionMeta TPortionMeta::SerializeToProto() const { +NKikimrTxColumnShard::TIndexPortionMeta TPortionMeta::SerializeToProto(const NPortion::EProduced produced) const { FullValidation(); NKikimrTxColumnShard::TIndexPortionMeta portionMeta; portionMeta.SetTierName(TierName); @@ -21,22 +21,22 @@ NKikimrTxColumnShard::TIndexPortionMeta TPortionMeta::SerializeToProto() const { portionMeta.SetColumnBlobBytes(ColumnBlobBytes); portionMeta.SetIndexRawBytes(IndexRawBytes); portionMeta.SetIndexBlobBytes(IndexBlobBytes); - switch (Produced) { - case TPortionMeta::EProduced::UNSPECIFIED: + switch (produced) { + case NPortion::EProduced::UNSPECIFIED: Y_ABORT_UNLESS(false); - case TPortionMeta::EProduced::INSERTED: + case NPortion::EProduced::INSERTED: portionMeta.SetIsInserted(true); break; - case TPortionMeta::EProduced::COMPACTED: + case NPortion::EProduced::COMPACTED: portionMeta.SetIsCompacted(true); break; - case TPortionMeta::EProduced::SPLIT_COMPACTED: + case NPortion::EProduced::SPLIT_COMPACTED: portionMeta.SetIsSplitCompacted(true); break; - case TPortionMeta::EProduced::EVICTED: + case NPortion::EProduced::EVICTED: portionMeta.SetIsEvicted(true); break; - case TPortionMeta::EProduced::INACTIVE: + case NPortion::EProduced::INACTIVE: Y_ABORT("Unexpected inactive case"); //portionMeta->SetInactive(true); break; @@ -59,7 +59,7 @@ NKikimrTxColumnShard::TIndexPortionMeta TPortionMeta::SerializeToProto() const { TString TPortionMeta::DebugString() const { TStringBuilder sb; - sb << "(produced=" << Produced << ";"; + sb << "("; if (TierName) { sb << "tier_name=" << TierName << ";"; } diff --git a/ydb/core/tx/columnshard/engines/portions/meta.h b/ydb/core/tx/columnshard/engines/portions/meta.h index bbb77c6672d5..d56619e99566 100644 --- a/ydb/core/tx/columnshard/engines/portions/meta.h +++ b/ydb/core/tx/columnshard/engines/portions/meta.h @@ -115,10 +115,6 @@ class TPortionMeta: public TPortionMetaBase { CompactionLevel = level; } - using EProduced = NPortion::EProduced; - - EProduced Produced = EProduced::UNSPECIFIED; - std::optional GetTierNameOptional() const; ui64 GetMetadataMemorySize() const { @@ -133,11 +129,7 @@ class TPortionMeta: public TPortionMetaBase { return sizeof(TPortionMeta) + FirstPKRow.GetDataSize() + LastPKRow.GetDataSize() + TBase::GetMetadataDataSize(); } - NKikimrTxColumnShard::TIndexPortionMeta SerializeToProto() const; - - EProduced GetProduced() const { - return Produced; - } + NKikimrTxColumnShard::TIndexPortionMeta SerializeToProto(const NPortion::EProduced produced) const; TString DebugString() const; }; diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp index c1b26908060e..809adabf12ef 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp @@ -60,7 +60,7 @@ void TPortionInfo::SerializeToProto(NKikimrColumnShardDataSharingProto::TPortion *proto.MutableRemoveSnapshot() = RemoveSnapshot.SerializeToProto(); } - *proto.MutableMeta() = Meta.SerializeToProto(); + *proto.MutableMeta() = Meta.SerializeToProto(GetProduced()); } TConclusionStatus TPortionInfo::DeserializeFromProto(const NKikimrColumnShardDataSharingProto::TPortionInfo& proto) { diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.h b/ydb/core/tx/columnshard/engines/portions/portion_info.h index eec5b4d59ec6..027e2b4f9da6 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.h +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.h @@ -310,7 +310,7 @@ class TPortionInfo { if (GetTierNameDef(NBlobOperations::TGlobal::DefaultStorageId) != NBlobOperations::TGlobal::DefaultStorageId) { return NPortion::EVICTED; } - return GetMeta().GetProduced(); + return GetPortionType() == EPortionType::Compacted ? NPortion::EProduced::SPLIT_COMPACTED : NPortion::EProduced::INSERTED; } bool ValidSnapshotInfo() const { diff --git a/ydb/core/tx/columnshard/engines/portions/written.cpp b/ydb/core/tx/columnshard/engines/portions/written.cpp index 5112a3c1e12b..5db336d99be1 100644 --- a/ydb/core/tx/columnshard/engines/portions/written.cpp +++ b/ydb/core/tx/columnshard/engines/portions/written.cpp @@ -7,7 +7,7 @@ namespace NKikimr::NOlap { void TWrittenPortionInfo::DoSaveMetaToDatabase(NIceDb::TNiceDb& db) const { - auto metaProto = GetMeta().SerializeToProto(); + auto metaProto = GetMeta().SerializeToProto(NPortion::EProduced::INSERTED); using IndexPortions = NColumnShard::Schema::IndexPortions; const auto removeSnapshot = GetRemoveSnapshotOptional(); AFL_VERIFY(InsertWriteId); diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/plain_read_data.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/plain_read_data.cpp index 3847a929dee6..2e537f55a2c2 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/plain_read_data.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/plain_read_data.cpp @@ -16,7 +16,7 @@ TPlainReadData::TPlainReadData(const std::shared_ptr& context) ui64 insertedPortionsBytes = 0; ui64 committedPortionsBytes = 0; for (auto&& i : portions) { - if (i->GetMeta().GetProduced() == NPortion::EProduced::COMPACTED || i->GetMeta().GetProduced() == NPortion::EProduced::SPLIT_COMPACTED) { + if (i->GetPortionType() == EPortionType::Compacted) { compactedPortionsBytes += i->GetTotalBlobBytes(); } else { insertedPortionsBytes += i->GetTotalBlobBytes(); diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/plain_read_data.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/plain_read_data.cpp index 898c6f92c5db..1d6f546e978f 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/plain_read_data.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/plain_read_data.cpp @@ -13,7 +13,7 @@ TPlainReadData::TPlainReadData(const std::shared_ptr& context) ui64 compactedPortionsBytes = 0; ui64 insertedPortionsBytes = 0; for (auto&& i : portions) { - if (i->GetMeta().GetProduced() == NPortion::EProduced::COMPACTED || i->GetMeta().GetProduced() == NPortion::EProduced::SPLIT_COMPACTED) { + if (i->GetPortionType() == EPortionType::Compacted) { compactedPortionsBytes += i->GetTotalBlobBytes(); } else { insertedPortionsBytes += i->GetTotalBlobBytes(); diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp index 664a4ac881cc..9ad6dd03c183 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp @@ -9,9 +9,9 @@ void TStatsIterator::AppendStats( const std::vector>& builders, const TPortionDataAccessor& portionPtr) const { const TPortionInfo& portion = portionPtr.GetPortionInfo(); auto portionSchema = ReadMetadata->GetLoadSchemaVerified(portion); - auto it = PortionType.find(portion.GetMeta().Produced); + auto it = PortionType.find(portion.GetProduced()); if (it == PortionType.end()) { - it = PortionType.emplace(portion.GetMeta().Produced, ::ToString(portion.GetMeta().Produced)).first; + it = PortionType.emplace(portion.GetProduced(), ::ToString(portion.GetProduced())).first; } const arrow::util::string_view prodView = it->second.GetView(); const bool activity = !portion.HasRemoveSnapshot(); diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/portions/portions.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/portions/portions.cpp index ae0e9bf3e29f..9cbd495169a0 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/portions/portions.cpp +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/portions/portions.cpp @@ -8,7 +8,7 @@ namespace NKikimr::NOlap::NReader::NSysView::NPortions { void TStatsIterator::AppendStats(const std::vector>& builders, const TPortionInfo& portion) const { NArrow::Append(*builders[0], portion.GetPathId().GetRawValue()); - const std::string prod = ::ToString(portion.GetMeta().Produced); + const std::string prod = ::ToString(portion.GetProduced()); NArrow::Append(*builders[1], prod); NArrow::Append(*builders[2], ReadMetadata->GetTabletId()); NArrow::Append(*builders[3], portion.GetRecordsCount()); diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp b/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp index b60b20f48136..049e0109fe81 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp @@ -368,7 +368,6 @@ TConclusion ISnapshotSchema::PrepareForWrite(c constructor.GetPortionConstructor().MutablePortionConstructor().AddMetadata(*this, deletionsCount, primaryKeys, std::nullopt); constructor.GetPortionConstructor().MutablePortionConstructor().MutableMeta().SetTierName(IStoragesManager::DefaultStorageId); constructor.GetPortionConstructor().MutablePortionConstructor().MutableMeta().SetCompactionLevel(0); - constructor.GetPortionConstructor().MutablePortionConstructor().MutableMeta().UpdateRecordsMeta(NPortion::EProduced::INSERTED); return TWritePortionInfoWithBlobsResult(std::move(constructor)); } diff --git a/ydb/core/tx/columnshard/engines/storage/granule/granule.h b/ydb/core/tx/columnshard/engines/storage/granule/granule.h index 571a94971c99..207305332fec 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/granule.h +++ b/ydb/core/tx/columnshard/engines/storage/granule/granule.h @@ -91,14 +91,14 @@ class TGranuleAdditiveSummary { } void AddPortion(const TPortionInfo& info) { - if (info.GetMeta().GetProduced() == NPortion::EProduced::INSERTED) { + if (info.GetPortionType() == EPortionType::Written) { Owner.Inserted.AddPortion(info); } else { Owner.Compacted.AddPortion(info); } } void RemovePortion(const TPortionInfo& info) { - if (info.GetMeta().GetProduced() == NPortion::EProduced::INSERTED) { + if (info.GetPortionType() == EPortionType::Written) { Owner.Inserted.RemovePortion(info); } else { Owner.Compacted.RemovePortion(info); diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h index 33707fd830c5..fd886db94103 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h @@ -595,10 +595,10 @@ class TPortionsPool { sb << "{" << "oldest=" << "(" << oldestPortion->IndexKeyStart().DebugString() << ":" << oldestPortion->IndexKeyEnd().DebugString() << ":" - << oldestPortion->RecordSnapshotMax().GetPlanStep() << ":" << oldestPortion->GetMeta().GetProduced() << ");" + << oldestPortion->RecordSnapshotMax().GetPlanStep() << ":" << oldestPortion->GetProduced() << ");" << "youngest=" << "(" << youngestPortion->IndexKeyStart().DebugString() << ":" << youngestPortion->IndexKeyEnd().DebugString() << ":" - << youngestPortion->RecordSnapshotMax().GetPlanStep() << ":" << youngestPortion->GetMeta().GetProduced() << ");" + << youngestPortion->RecordSnapshotMax().GetPlanStep() << ":" << youngestPortion->GetProduced() << ");" << "}"; return sb; } else { @@ -1187,7 +1187,7 @@ class TPortionBuckets { if (itFrom == Buckets.end()) { const TDuration freshness = now - TInstant::MilliSeconds(portion->RecordSnapshotMax().GetPlanStep()); if (Y_LIKELY(!NYDBTest::TControllers::GetColumnShardController()->NeedForceCompactionBacketsConstruction())) { - if (freshness < GetCommonFreshnessCheckDuration() || portion->GetMeta().GetProduced() == NPortion::EProduced::INSERTED) { + if (freshness < GetCommonFreshnessCheckDuration() || portion->GetProduced() == NPortion::EProduced::INSERTED) { AddOther(portion, now); return; } diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.h index c2d9f19e00b8..9f7b24243519 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.h @@ -103,7 +103,7 @@ class TOptimizerPlanner: public IOptimizerPlanner { if (i->GetCompactionLevel()) { continue; } - if (i->GetTotalBlobBytes() > 512 * 1024 && i->GetMeta().GetProduced() != NPortion::EProduced::INSERTED) { + if (i->GetTotalBlobBytes() > 512 * 1024 && i->GetPortionType() == EPortionType::Compacted) { for (i32 levelIdx = Levels.size() - 1; levelIdx >= 0; --levelIdx) { if (Levels[levelIdx]->CanTakePortion(i)) { Levels[levelIdx]->ModifyPortions({ i }, {}); diff --git a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp index da09159e681c..fc85ba5ab143 100644 --- a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp +++ b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp @@ -118,7 +118,10 @@ class TTestDbWrapper: public IDbWrapper { const std::function&&, const NKikimrTxColumnShard::TIndexPortionMeta&)>& callback) override { for (auto&& i : Portions) { if (!pathId || *pathId == i.second->GetPathId()) { - callback(i.second->BuildConstructor(false, false), i.second->GetMeta().SerializeToProto()); + callback(i.second->BuildConstructor(false, false), + i.second->GetMeta().SerializeToProto(i.second->GetPortionType() == NOlap::EPortionType::Compacted + ? NPortion::EProduced::SPLIT_COMPACTED + : NPortion::EProduced::INSERTED)); } } return true; @@ -127,7 +130,9 @@ class TTestDbWrapper: public IDbWrapper { void WriteColumn(const TPortionInfo& portion, const TColumnRecord& row, const ui32 firstPKColumnId) override { auto rowProto = row.GetMeta().SerializeToProto(); if (firstPKColumnId == row.GetColumnId() && row.GetChunkIdx() == 0) { - *rowProto.MutablePortionMeta() = portion.GetMeta().SerializeToProto(); + *rowProto.MutablePortionMeta() = portion.GetMeta().SerializeToProto(portion.GetPortionType() == NOlap::EPortionType::Compacted + ? NPortion::EProduced::SPLIT_COMPACTED + : NPortion::EProduced::INSERTED); } auto& data = Indices[0].Columns[portion.GetPathId()];