Skip to content

Commit e958419

Browse files
protobuf-github-botcopybara-github
protobuf-github-bot
authored and
copybara-github
committed
Use __builtin_expect_with_probability for proto field presence checks.
PiperOrigin-RevId: 721817344
1 parent 2141d1c commit e958419

File tree

6 files changed

+145
-41
lines changed

6 files changed

+145
-41
lines changed

‎src/google/protobuf/compiler/cpp/helpers.cc‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1042,9 +1042,26 @@ bool IsLikelyPresent(const FieldDescriptor* field, const Options& options) {
10421042
return false;
10431043
}
10441044

1045-
float GetPresenceProbability(const FieldDescriptor* field,
1046-
const Options& options) {
1047-
return 1.f;
1045+
absl::optional<float> GetPresenceProbability(const FieldDescriptor* field,
1046+
const Options& options) {
1047+
return absl::nullopt;
1048+
}
1049+
1050+
absl::optional<float> GetFieldGroupPresenceProbability(
1051+
const std::vector<const FieldDescriptor*>& fields, const Options& options) {
1052+
ABSL_DCHECK(!fields.empty());
1053+
if (!IsProfileDriven(options)) return absl::nullopt;
1054+
1055+
double all_absent_probability = 1.0;
1056+
1057+
for (const auto* field : fields) {
1058+
absl::optional<float> probability = GetPresenceProbability(field, options);
1059+
if (!probability) {
1060+
return absl::nullopt;
1061+
}
1062+
all_absent_probability *= 1.0 - *probability;
1063+
}
1064+
return 1.0 - all_absent_probability;
10481065
}
10491066

10501067
bool IsStringInliningEnabled(const Options& options) {

‎src/google/protobuf/compiler/cpp/helpers.h‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -366,8 +366,15 @@ bool IsRarelyPresent(const FieldDescriptor* field, const Options& options);
366366
// Returns true if `field` is likely to be present based on PDProto profile.
367367
bool IsLikelyPresent(const FieldDescriptor* field, const Options& options);
368368

369-
float GetPresenceProbability(const FieldDescriptor* field,
370-
const Options& options);
369+
absl::optional<float> GetPresenceProbability(const FieldDescriptor* field,
370+
const Options& options);
371+
372+
// GetFieldGroupPresenceProbability computes presence probability for a group of
373+
// fields. It uses the absence probability (easier to compute)
374+
// (1 - p1) * (1 - p2) * ... * (1 - pn), and in the end the aggregate presence
375+
// probability can be expressed as (1 - all_absent_probability).
376+
absl::optional<float> GetFieldGroupPresenceProbability(
377+
const std::vector<const FieldDescriptor*>& fields, const Options& options);
371378

372379
bool IsStringInliningEnabled(const Options& options);
373380

‎src/google/protobuf/compiler/cpp/message.cc‎

Lines changed: 98 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -95,9 +95,71 @@ std::string ConditionalToCheckBitmasks(
9595
return result + (return_success ? " == 0" : " != 0");
9696
}
9797

98+
template <typename PredicateT>
99+
void DebugAssertUniform(const std::vector<const FieldDescriptor*>& fields,
100+
const Options& options, PredicateT&& pred) {
101+
ABSL_DCHECK(!fields.empty() && absl::c_all_of(fields, [&](const auto* f) {
102+
return pred(f) == pred(fields.front());
103+
}));
104+
}
105+
106+
void DebugAssertUniformLikelyPresence(
107+
const std::vector<const FieldDescriptor*>& fields, const Options& options) {
108+
DebugAssertUniform(fields, options, [&](const FieldDescriptor* f) {
109+
return IsLikelyPresent(f, options);
110+
});
111+
}
112+
113+
// Generates a condition that checks presence of a field. If probability is
114+
// provided, the condition will be wrapped with
115+
// PROTOBUF_EXPECT_TRUE_WITH_PROBABILITY.
116+
//
117+
// If use_cached_has_bits is true, the condition will be generated based on
118+
// cached_has_bits. Otherwise, the condition will be generated based on the
119+
// _has_bits_ array, with has_array_index indicating which element of the array
120+
// to use.
121+
std::string GenerateConditionMaybeWithProbability(
122+
uint32_t mask, absl::optional<float> probability, bool use_cached_has_bits,
123+
absl::optional<int> has_array_index) {
124+
std::string condition;
125+
if (use_cached_has_bits) {
126+
condition = absl::StrFormat("(cached_has_bits & 0x%08xu) != 0", mask);
127+
} else {
128+
// We only use has_array_index when use_cached_has_bits is false, make sure
129+
// we pas a valid index when we need it.
130+
ABSL_DCHECK(has_array_index.has_value());
131+
condition = absl::StrFormat("(this_._impl_._has_bits_[%d] & 0x%08xu) != 0",
132+
*has_array_index, mask);
133+
}
134+
if (probability.has_value()) {
135+
return absl::StrFormat("PROTOBUF_EXPECT_TRUE_WITH_PROBABILITY(%s, %.3f)",
136+
condition, *probability);
137+
}
138+
return condition;
139+
}
140+
141+
std::string GenerateConditionMaybeWithProbabilityForField(
142+
int has_bit_index, const FieldDescriptor* field, const Options& options) {
143+
auto prob = GetPresenceProbability(field, options);
144+
return GenerateConditionMaybeWithProbability(
145+
1u << (has_bit_index % 32), prob,
146+
/*use_cached_has_bits*/ true,
147+
/*has_array_index*/ absl::nullopt);
148+
}
149+
150+
std::string GenerateConditionMaybeWithProbabilityForGroup(
151+
uint32_t mask, const std::vector<const FieldDescriptor*>& fields,
152+
const Options& options) {
153+
auto prob = GetFieldGroupPresenceProbability(fields, options);
154+
return GenerateConditionMaybeWithProbability(
155+
mask, prob,
156+
/*use_cached_has_bits*/ true,
157+
/*has_array_index*/ absl::nullopt);
158+
}
159+
98160
void PrintPresenceCheck(const FieldDescriptor* field,
99161
const std::vector<int>& has_bit_indices, io::Printer* p,
100-
int* cached_has_word_index) {
162+
int* cached_has_word_index, const Options& options) {
101163
if (!field->options().weak()) {
102164
int has_bit_index = has_bit_indices[field->index()];
103165
if (*cached_has_word_index != (has_bit_index / 32)) {
@@ -107,9 +169,10 @@ void PrintPresenceCheck(const FieldDescriptor* field,
107169
cached_has_bits = $has_bits$[$index$];
108170
)cc");
109171
}
110-
p->Emit({{"mask", absl::StrFormat("0x%08xu", 1u << (has_bit_index % 32))}},
172+
p->Emit({{"condition", GenerateConditionMaybeWithProbabilityForField(
173+
has_bit_index, field, options)}},
111174
R"cc(
112-
if ((cached_has_bits & $mask$) != 0) {
175+
if ($condition$) {
113176
)cc");
114177
} else {
115178
p->Emit(R"cc(
@@ -1319,8 +1382,8 @@ void MessageGenerator::EmitCheckAndUpdateByteSizeForField(
13191382
}
13201383

13211384
int has_bit_index = has_bit_indices_[field->index()];
1322-
p->Emit({{"mask",
1323-
absl::StrFormat("0x%08xu", uint32_t{1} << (has_bit_index % 32))},
1385+
p->Emit({{"condition", GenerateConditionMaybeWithProbabilityForField(
1386+
has_bit_index, field, options_)},
13241387
{"check_nondefault_and_emit_body",
13251388
[&] {
13261389
// Note that it's possible that the field has explicit presence.
@@ -1330,7 +1393,7 @@ void MessageGenerator::EmitCheckAndUpdateByteSizeForField(
13301393
/*with_enclosing_braces_always=*/false);
13311394
}}},
13321395
R"cc(
1333-
if ((cached_has_bits & $mask$) != 0) {
1396+
if ($condition$) {
13341397
$check_nondefault_and_emit_body$;
13351398
}
13361399
)cc");
@@ -3186,9 +3249,10 @@ void MessageGenerator::GenerateCopyInitFields(io::Printer* p) const {
31863249
if (has_bit_indices_.empty()) {
31873250
p->Emit("from.$field$ != nullptr");
31883251
} else {
3189-
int index = has_bit_indices_[field->index()];
3190-
std::string mask = absl::StrFormat("0x%08xu", 1u << (index % 32));
3191-
p->Emit({{"mask", mask}}, "(cached_has_bits & $mask$) != 0");
3252+
int has_bit_index = has_bit_indices_[field->index()];
3253+
p->Emit({{"condition", GenerateConditionMaybeWithProbabilityForField(
3254+
has_bit_index, field, options_)}},
3255+
"$condition$");
31923256
}
31933257
};
31943258

@@ -3603,11 +3667,11 @@ void MessageGenerator::GenerateClear(io::Printer* p) {
36033667
!IsLikelyPresent(fields.back(), options_) &&
36043668
(memset_end != fields.back() || merge_zero_init);
36053669

3670+
DebugAssertUniformLikelyPresence(fields, options_);
3671+
36063672
if (check_has_byte) {
36073673
// Emit an if() that will let us skip the whole chunk if none are set.
36083674
uint32_t chunk_mask = GenChunkMask(fields, has_bit_indices_);
3609-
std::string chunk_mask_str =
3610-
absl::StrCat(absl::Hex(chunk_mask, absl::kZeroPad8));
36113675

36123676
// Check (up to) 8 has_bits at a time if we have more than one field in
36133677
// this chunk. Due to field layout ordering, we may check
@@ -3619,7 +3683,9 @@ void MessageGenerator::GenerateClear(io::Printer* p) {
36193683
cached_has_word_index = HasWordIndex(fields.front());
36203684
format("cached_has_bits = $has_bits$[$1$];\n", cached_has_word_index);
36213685
}
3622-
format("if ((cached_has_bits & 0x$1$u) != 0) {\n", chunk_mask_str);
3686+
3687+
format("if ($1$) {\n", GenerateConditionMaybeWithProbabilityForGroup(
3688+
chunk_mask, fields, options_));
36233689
format.Indent();
36243690
}
36253691

@@ -3652,8 +3718,8 @@ void MessageGenerator::GenerateClear(io::Printer* p) {
36523718
field->cpp_type() == FieldDescriptor::CPPTYPE_STRING);
36533719

36543720
if (have_enclosing_if) {
3655-
PrintPresenceCheck(field, has_bit_indices_, p,
3656-
&cached_has_word_index);
3721+
PrintPresenceCheck(field, has_bit_indices_, p, &cached_has_word_index,
3722+
options_);
36573723
format.Indent();
36583724
}
36593725

@@ -4266,6 +4332,8 @@ void MessageGenerator::GenerateClassSpecificMergeImpl(io::Printer* p) {
42664332
const bool check_has_byte = cache_has_bits && fields.size() > 1 &&
42674333
!IsLikelyPresent(fields.back(), options_);
42684334

4335+
DebugAssertUniformLikelyPresence(fields, options_);
4336+
42694337
if (cache_has_bits &&
42704338
cached_has_word_index != HasWordIndex(fields.front())) {
42714339
cached_has_word_index = HasWordIndex(fields.front());
@@ -4276,16 +4344,15 @@ void MessageGenerator::GenerateClassSpecificMergeImpl(io::Printer* p) {
42764344
if (check_has_byte) {
42774345
// Emit an if() that will let us skip the whole chunk if none are set.
42784346
uint32_t chunk_mask = GenChunkMask(fields, has_bit_indices_);
4279-
std::string chunk_mask_str =
4280-
absl::StrCat(absl::Hex(chunk_mask, absl::kZeroPad8));
42814347

42824348
// Check (up to) 8 has_bits at a time if we have more than one field in
42834349
// this chunk. Due to field layout ordering, we may check
42844350
// _has_bits_[last_chunk * 8 / 32] multiple times.
42854351
ABSL_DCHECK_LE(2, popcnt(chunk_mask));
42864352
ABSL_DCHECK_GE(8, popcnt(chunk_mask));
42874353

4288-
format("if ((cached_has_bits & 0x$1$u) != 0) {\n", chunk_mask_str);
4354+
format("if ($1$) {\n", GenerateConditionMaybeWithProbabilityForGroup(
4355+
chunk_mask, fields, options_));
42894356
format.Indent();
42904357
}
42914358

@@ -4317,9 +4384,8 @@ void MessageGenerator::GenerateClassSpecificMergeImpl(io::Printer* p) {
43174384
// Check hasbit, using cached bits.
43184385
ABSL_CHECK(HasHasbit(field));
43194386
int has_bit_index = has_bit_indices_[field->index()];
4320-
const std::string mask = absl::StrCat(
4321-
absl::Hex(1u << (has_bit_index % 32), absl::kZeroPad8));
4322-
format("if ((cached_has_bits & 0x$1$u) != 0) {\n", mask);
4387+
format("if ($1$) {\n", GenerateConditionMaybeWithProbabilityForField(
4388+
has_bit_index, field, options_));
43234389
format.Indent();
43244390

43254391
if (GetFieldHasbitMode(field) == HasbitMode::kHintHasbit) {
@@ -4556,6 +4622,9 @@ void MessageGenerator::GenerateSerializeOneField(io::Printer* p,
45564622

45574623
PrintFieldComment(Formatter{p}, field, options_);
45584624
if (HasHasbit(field)) {
4625+
int has_bit_index = HasBitIndex(field);
4626+
int has_word_index = has_bit_index / 32;
4627+
bool use_cached_has_bits = cached_has_bits_index == has_word_index;
45594628
p->Emit(
45604629
{
45614630
{"body",
@@ -4564,18 +4633,10 @@ void MessageGenerator::GenerateSerializeOneField(io::Printer* p,
45644633
std::move(emit_body),
45654634
/*with_enclosing_braces_always=*/false);
45664635
}},
4567-
{"cond",
4568-
[&] {
4569-
int has_bit_index = HasBitIndex(field);
4570-
auto v = p->WithVars(HasBitVars(field));
4571-
// Attempt to use the state of cached_has_bits, if possible.
4572-
if (cached_has_bits_index == has_bit_index / 32) {
4573-
p->Emit("(cached_has_bits & $has_mask$) != 0");
4574-
} else {
4575-
p->Emit(
4576-
"(this_.$has_bits$[$has_array_index$] & $has_mask$) != 0");
4577-
}
4578-
}},
4636+
{"cond", GenerateConditionMaybeWithProbability(
4637+
1u << (has_bit_index % 32),
4638+
GetPresenceProbability(field, options_),
4639+
use_cached_has_bits, has_word_index)},
45794640
},
45804641
R"cc(
45814642
if ($cond$) {
@@ -5188,6 +5249,7 @@ void MessageGenerator::GenerateByteSize(io::Printer* p) {
51885249
const bool check_has_byte =
51895250
fields.size() > 1 && HasWordIndex(fields[0]) != kNoHasbit &&
51905251
!IsLikelyPresent(fields.back(), options_);
5252+
DebugAssertUniformLikelyPresence(fields, options_);
51915253
p->Emit(
51925254
{{"update_byte_size_for_chunk",
51935255
[&] {
@@ -5221,9 +5283,10 @@ void MessageGenerator::GenerateByteSize(io::Printer* p) {
52215283
ABSL_DCHECK_LE(2, popcnt(chunk_mask));
52225284
ABSL_DCHECK_GE(8, popcnt(chunk_mask));
52235285

5224-
p->Emit(
5225-
{{"mask", absl::StrFormat("0x%08xu", chunk_mask)}},
5226-
"if ((cached_has_bits & $mask$) != 0)");
5286+
p->Emit({{"condition",
5287+
GenerateConditionMaybeWithProbabilityForGroup(
5288+
chunk_mask, fields, options_)}},
5289+
"if ($condition$)");
52275290
}}},
52285291
R"cc(
52295292
$may_update_cached_has_word_index$;

‎src/google/protobuf/compiler/cpp/parse_function_generator.cc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,9 @@ ParseFunctionGenerator::ParseFunctionGenerator(
7474
fields.push_back({
7575
field,
7676
index < has_bit_indices.size() ? has_bit_indices[index] : -1,
77-
GetPresenceProbability(field, options_),
77+
// When not present, we're not sure how likely "field" is present.
78+
// Assign a 50% probability to avoid pessimizing it.
79+
GetPresenceProbability(field, options_).value_or(0.5f),
7880
GetLazyStyle(field, options_, scc_analyzer_),
7981
IsStringInlined(field, options_),
8082
IsImplicitWeakField(field, options_, scc_analyzer_),

‎src/google/protobuf/port_def.inc‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -890,6 +890,20 @@ namespace internal {
890890
#endif // !NDEBUG
891891
#endif // has_builtin(__builtin_assume)
892892

893+
// PROTOBUF_EXPECT_TRUE_WITH_PROBABILITY(expr, prob) tells the compiler that
894+
// the expression expr is likely to be true with probability prob. This is
895+
// helpful for the compiler to make better optimization decisions when we know
896+
// the probability of a certain branch (e.g. from a profile).
897+
#ifdef PROTOBUF_EXPECT_TRUE_WITH_PROBABILITY
898+
#error PROTOBUF_EXPECT_TRUE_WITH_PROBABILITY was previously defined
899+
#endif
900+
#if ABSL_HAVE_BUILTIN(__builtin_expect_with_probability)
901+
#define PROTOBUF_EXPECT_TRUE_WITH_PROBABILITY(expr, prob) \
902+
__builtin_expect_with_probability((expr), 1, (prob))
903+
#else
904+
#define PROTOBUF_EXPECT_TRUE_WITH_PROBABILITY(expr, prob) (expr)
905+
#endif
906+
893907
#ifdef PROTOBUF_DEPRECATE_AND_INLINE
894908
#error PROTOBUF_DEPRECATE_AND_INLINE was previously defined
895909
#endif

‎src/google/protobuf/port_undef.inc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
#undef PROTOBUF_RESTRICT
3737
#undef PROTOBUF_UNUSED
3838
#undef PROTOBUF_ASSUME
39+
#undef PROTOBUF_EXPECT_TRUE_WITH_PROBABILITY
3940
#undef PROTOBUF_DEPRECATE_AND_INLINE
4041
#undef PROTOBUF_EXPORT_TEMPLATE_DECLARE
4142
#undef PROTOBUF_EXPORT_TEMPLATE_DEFINE

0 commit comments

Comments
 (0)