Skip to content

Commit fcf0a01

Browse files
surbhigarg92surbhigarg92
authored
feat: Support Exemplar (#3997)
This PR adds support for Exemplars in Client side metrics. Exemplars will also include `x-goog-spanner-request-id` attribute
1 parent 2a439dc commit fcf0a01

10 files changed

+152
-49
lines changed

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616

1717
package com.google.cloud.spanner;
1818

19+
import static com.google.cloud.spanner.XGoogSpannerRequestId.REQUEST_ID;
20+
1921
import com.google.api.core.InternalApi;
2022
import com.google.api.gax.tracing.OpenTelemetryMetricsRecorder;
2123
import com.google.common.collect.ImmutableList;
@@ -26,7 +28,9 @@
2628
import io.opentelemetry.sdk.metrics.InstrumentSelector;
2729
import io.opentelemetry.sdk.metrics.InstrumentType;
2830
import io.opentelemetry.sdk.metrics.View;
31+
import java.util.Arrays;
2932
import java.util.Collection;
33+
import java.util.HashSet;
3034
import java.util.List;
3135
import java.util.Map;
3236
import java.util.Set;
@@ -94,6 +98,8 @@ public class BuiltInMetricsConstant {
9498
AttributeKey.stringKey("directpath_enabled");
9599
public static final AttributeKey<String> DIRECT_PATH_USED_KEY =
96100
AttributeKey.stringKey("directpath_used");
101+
public static final AttributeKey<String> REQUEST_ID_KEY = AttributeKey.stringKey(REQUEST_ID);
102+
public static Set<String> ALLOWED_EXEMPLARS_ATTRIBUTES = new HashSet<>(Arrays.asList(REQUEST_ID));
97103

98104
// IP address prefixes allocated for DirectPath backends.
99105
public static final String DP_IPV6_PREFIX = "2001:4860:8040";
@@ -168,6 +174,7 @@ static Map<InstrumentSelector, View> getAllViews() {
168174
Aggregation.sum(),
169175
InstrumentType.COUNTER,
170176
"1");
177+
defineSpannerView(views);
171178
defineGRPCView(views);
172179
return views.build();
173180
}
@@ -200,6 +207,19 @@ private static void defineView(
200207
viewMap.put(selector, view);
201208
}
202209

210+
private static void defineSpannerView(ImmutableMap.Builder<InstrumentSelector, View> viewMap) {
211+
InstrumentSelector selector =
212+
InstrumentSelector.builder()
213+
.setMeterName(BuiltInMetricsConstant.SPANNER_METER_NAME)
214+
.build();
215+
Set<String> attributesFilter =
216+
BuiltInMetricsConstant.COMMON_ATTRIBUTES.stream()
217+
.map(AttributeKey::getKey)
218+
.collect(Collectors.toSet());
219+
View view = View.builder().setAttributeFilter(attributesFilter).build();
220+
viewMap.put(selector, view);
221+
}
222+
203223
private static void defineGRPCView(ImmutableMap.Builder<InstrumentSelector, View> viewMap) {
204224
for (String metric : BuiltInMetricsConstant.GRPC_METRICS_TO_ENABLE) {
205225
InstrumentSelector selector =

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsTracer.java

Lines changed: 38 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -39,14 +39,21 @@ class BuiltInMetricsTracer extends MetricsTracer implements ApiTracer {
3939
private final Map<String, String> attributes = new HashMap<>();
4040
private Float gfeLatency = null;
4141
private Float afeLatency = null;
42+
private TraceWrapper traceWrapper;
4243
private long gfeHeaderMissingCount = 0;
4344
private long afeHeaderMissingCount = 0;
45+
private final ISpan currentSpan;
4446

4547
BuiltInMetricsTracer(
46-
MethodName methodName, BuiltInMetricsRecorder builtInOpenTelemetryMetricsRecorder) {
48+
MethodName methodName,
49+
BuiltInMetricsRecorder builtInOpenTelemetryMetricsRecorder,
50+
TraceWrapper traceWrapper,
51+
ISpan currentSpan) {
4752
super(methodName, builtInOpenTelemetryMetricsRecorder);
4853
this.builtInOpenTelemetryMetricsRecorder = builtInOpenTelemetryMetricsRecorder;
4954
this.attributes.put(METHOD_ATTRIBUTE, methodName.toString());
55+
this.traceWrapper = traceWrapper;
56+
this.currentSpan = currentSpan;
5057
}
5158

5259
/**
@@ -55,10 +62,12 @@ class BuiltInMetricsTracer extends MetricsTracer implements ApiTracer {
5562
*/
5663
@Override
5764
public void attemptSucceeded() {
58-
super.attemptSucceeded();
59-
attributes.put(STATUS_ATTRIBUTE, StatusCode.Code.OK.toString());
60-
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
61-
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
65+
try (IScope s = this.traceWrapper.withSpan(this.currentSpan)) {
66+
super.attemptSucceeded();
67+
attributes.put(STATUS_ATTRIBUTE, StatusCode.Code.OK.toString());
68+
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
69+
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
70+
}
6271
}
6372

6473
/**
@@ -67,10 +76,12 @@ public void attemptSucceeded() {
6776
*/
6877
@Override
6978
public void attemptCancelled() {
70-
super.attemptCancelled();
71-
attributes.put(STATUS_ATTRIBUTE, StatusCode.Code.CANCELLED.toString());
72-
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
73-
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
79+
try (IScope s = this.traceWrapper.withSpan(this.currentSpan)) {
80+
super.attemptCancelled();
81+
attributes.put(STATUS_ATTRIBUTE, StatusCode.Code.CANCELLED.toString());
82+
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
83+
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
84+
}
7485
}
7586

7687
/**
@@ -83,10 +94,12 @@ public void attemptCancelled() {
8394
*/
8495
@Override
8596
public void attemptFailedDuration(Throwable error, java.time.Duration delay) {
86-
super.attemptFailedDuration(error, delay);
87-
attributes.put(STATUS_ATTRIBUTE, extractStatus(error));
88-
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
89-
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
97+
try (IScope s = this.traceWrapper.withSpan(this.currentSpan)) {
98+
super.attemptFailedDuration(error, delay);
99+
attributes.put(STATUS_ATTRIBUTE, extractStatus(error));
100+
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
101+
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
102+
}
90103
}
91104

92105
/**
@@ -98,10 +111,12 @@ public void attemptFailedDuration(Throwable error, java.time.Duration delay) {
98111
*/
99112
@Override
100113
public void attemptFailedRetriesExhausted(Throwable error) {
101-
super.attemptFailedRetriesExhausted(error);
102-
attributes.put(STATUS_ATTRIBUTE, extractStatus(error));
103-
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
104-
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
114+
try (IScope s = this.traceWrapper.withSpan(this.currentSpan)) {
115+
super.attemptFailedRetriesExhausted(error);
116+
attributes.put(STATUS_ATTRIBUTE, extractStatus(error));
117+
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
118+
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
119+
}
105120
}
106121

107122
/**
@@ -113,10 +128,12 @@ public void attemptFailedRetriesExhausted(Throwable error) {
113128
*/
114129
@Override
115130
public void attemptPermanentFailure(Throwable error) {
116-
super.attemptPermanentFailure(error);
117-
attributes.put(STATUS_ATTRIBUTE, extractStatus(error));
118-
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
119-
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
131+
try (IScope s = this.traceWrapper.withSpan(this.currentSpan)) {
132+
super.attemptPermanentFailure(error);
133+
attributes.put(STATUS_ATTRIBUTE, extractStatus(error));
134+
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
135+
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
136+
}
120137
}
121138

122139
void recordGFELatency(Float gfeLatency) {
@@ -140,7 +157,6 @@ public void addAttributes(Map<String, String> attributes) {
140157
super.addAttributes(attributes);
141158
this.attributes.putAll(attributes);
142159
}
143-
;
144160

145161
@Override
146162
public void addAttributes(String key, String value) {

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsTracerFactory.java

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,24 +38,31 @@ class BuiltInMetricsTracerFactory extends MetricsTracerFactory {
3838

3939
protected BuiltInMetricsRecorder builtInMetricsRecorder;
4040
private final Map<String, String> attributes;
41+
private final TraceWrapper traceWrapper;
4142

4243
/**
4344
* Pass in a Map of client level attributes which will be added to every single MetricsTracer
4445
* created from the ApiTracerFactory.
4546
*/
4647
public BuiltInMetricsTracerFactory(
47-
BuiltInMetricsRecorder builtInMetricsRecorder, Map<String, String> attributes) {
48+
BuiltInMetricsRecorder builtInMetricsRecorder,
49+
Map<String, String> attributes,
50+
TraceWrapper traceWrapper) {
4851
super(builtInMetricsRecorder, attributes);
4952
this.builtInMetricsRecorder = builtInMetricsRecorder;
5053
this.attributes = ImmutableMap.copyOf(attributes);
54+
this.traceWrapper = traceWrapper;
5155
}
5256

5357
@Override
5458
public ApiTracer newTracer(ApiTracer parent, SpanName spanName, OperationType operationType) {
59+
ISpan currentSpan = this.traceWrapper.getCurrentSpan();
5560
BuiltInMetricsTracer metricsTracer =
5661
new BuiltInMetricsTracer(
5762
MethodName.of(spanName.getClientName(), spanName.getMethodName()),
58-
builtInMetricsRecorder);
63+
builtInMetricsRecorder,
64+
this.traceWrapper,
65+
currentSpan);
5966
metricsTracer.addAttributes(attributes);
6067
return metricsTracer;
6168
}

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporterUtils.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import static com.google.api.MetricDescriptor.ValueType.DISTRIBUTION;
2323
import static com.google.api.MetricDescriptor.ValueType.DOUBLE;
2424
import static com.google.api.MetricDescriptor.ValueType.INT64;
25+
import static com.google.cloud.spanner.BuiltInMetricsConstant.ALLOWED_EXEMPLARS_ATTRIBUTES;
2526
import static com.google.cloud.spanner.BuiltInMetricsConstant.GAX_METER_NAME;
2627
import static com.google.cloud.spanner.BuiltInMetricsConstant.GRPC_METER_NAME;
2728
import static com.google.cloud.spanner.BuiltInMetricsConstant.PROJECT_ID_KEY;
@@ -293,7 +294,13 @@ private static String makeSpanName(String projectId, String traceId, String span
293294

294295
private static DroppedLabels mapFilteredAttributes(Attributes attributes) {
295296
DroppedLabels.Builder labels = DroppedLabels.newBuilder();
296-
attributes.forEach((k, v) -> labels.putLabel(cleanAttributeKey(k.getKey()), v.toString()));
297+
attributes.forEach(
298+
(k, v) -> {
299+
String key = cleanAttributeKey(k.getKey());
300+
if (ALLOWED_EXEMPLARS_ATTRIBUTES.contains(key)) {
301+
labels.putLabel(key, v.toString());
302+
}
303+
});
297304
return labels.build();
298305
}
299306

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@
7878
import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
7979
import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder;
8080
import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
81+
import io.opencensus.trace.Tracing;
8182
import io.opentelemetry.api.GlobalOpenTelemetry;
8283
import io.opentelemetry.api.OpenTelemetry;
8384
import io.opentelemetry.api.common.Attributes;
@@ -2073,7 +2074,15 @@ private ApiTracerFactory createMetricsApiTracerFactory() {
20732074
return openTelemetry != null
20742075
? new BuiltInMetricsTracerFactory(
20752076
new BuiltInMetricsRecorder(openTelemetry, BuiltInMetricsConstant.METER_NAME),
2076-
new HashMap<>())
2077+
new HashMap<>(),
2078+
new TraceWrapper(
2079+
Tracing.getTracer(),
2080+
// Using the OpenTelemetry object set in Spanner Options, will be NoOp if not set
2081+
this.getOpenTelemetry()
2082+
.getTracer(
2083+
MetricRegistryConstants.INSTRUMENTATION_SCOPE,
2084+
GaxProperties.getLibraryVersion(getClass())),
2085+
true))
20772086
: null;
20782087
}
20792088

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/XGoogSpannerRequestId.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,9 @@ public class XGoogSpannerRequestId {
3535
@VisibleForTesting
3636
static final String RAND_PROCESS_ID = XGoogSpannerRequestId.generateRandProcessId();
3737

38+
static String REQUEST_ID = "x-goog-spanner-request-id";
3839
public static final Metadata.Key<String> REQUEST_HEADER_KEY =
39-
Metadata.Key.of("x-goog-spanner-request-id", Metadata.ASCII_STRING_MARSHALLER);
40+
Metadata.Key.of(REQUEST_ID, Metadata.ASCII_STRING_MARSHALLER);
4041

4142
@VisibleForTesting
4243
static final long VERSION = 1; // The version of the specification being implemented.

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,7 @@
2424
import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.SPANNER_GFE_LATENCY;
2525

2626
import com.google.api.gax.tracing.ApiTracer;
27-
import com.google.cloud.spanner.BuiltInMetricsConstant;
28-
import com.google.cloud.spanner.CompositeTracer;
29-
import com.google.cloud.spanner.SpannerExceptionFactory;
30-
import com.google.cloud.spanner.SpannerRpcMetrics;
27+
import com.google.cloud.spanner.*;
3128
import com.google.common.cache.Cache;
3229
import com.google.common.cache.CacheBuilder;
3330
import com.google.spanner.admin.database.v1.DatabaseName;
@@ -120,6 +117,8 @@ public void start(Listener<RespT> responseListener, Metadata headers) {
120117
getMetricAttributes(key, method.getFullMethodName(), databaseName);
121118
Map<String, String> builtInMetricsAttributes =
122119
getBuiltInMetricAttributes(key, databaseName);
120+
builtInMetricsAttributes.put(
121+
BuiltInMetricsConstant.REQUEST_ID_KEY.getKey(), extractRequestId(headers));
123122
addBuiltInMetricAttributes(compositeTracer, builtInMetricsAttributes);
124123
super.start(
125124
new SimpleForwardingClientCallListener<RespT>(responseListener) {
@@ -128,6 +127,7 @@ public void onHeaders(Metadata metadata) {
128127
Boolean isDirectPathUsed =
129128
isDirectPathUsed(getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR));
130129
addDirectPathUsedAttribute(compositeTracer, isDirectPathUsed);
130+
131131
processHeader(
132132
metadata, tagContext, attributes, span, compositeTracer, isDirectPathUsed);
133133
super.onHeaders(metadata);
@@ -248,6 +248,10 @@ private DatabaseName extractDatabaseName(Metadata headers) throws ExecutionExcep
248248
return UNDEFINED_DATABASE_NAME;
249249
}
250250

251+
private String extractRequestId(Metadata headers) throws ExecutionException {
252+
return headers.get(XGoogSpannerRequestId.REQUEST_HEADER_KEY);
253+
}
254+
251255
private TagContext getTagContext(String key, String method, DatabaseName databaseName)
252256
throws ExecutionException {
253257
return tagsCache.get(

‎google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import static org.junit.Assert.fail;
2727
import static org.junit.Assume.assumeFalse;
2828

29+
import com.google.api.gax.core.GaxProperties;
2930
import com.google.api.gax.longrunning.OperationTimedPollAlgorithm;
3031
import com.google.api.gax.retrying.RetrySettings;
3132
import com.google.api.gax.tracing.ApiTracerFactory;
@@ -40,6 +41,7 @@
4041
import io.grpc.Server;
4142
import io.grpc.Status;
4243
import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder;
44+
import io.opencensus.trace.Tracing;
4345
import io.opentelemetry.api.OpenTelemetry;
4446
import io.opentelemetry.api.common.Attributes;
4547
import io.opentelemetry.sdk.OpenTelemetrySdk;
@@ -96,7 +98,14 @@ public ApiTracerFactory createMetricsTracerFactory() {
9698
OpenTelemetrySdk.builder().setMeterProvider(meterProvider.build()).build();
9799

98100
return new BuiltInMetricsTracerFactory(
99-
new BuiltInMetricsRecorder(openTelemetry, BuiltInMetricsConstant.METER_NAME), attributes);
101+
new BuiltInMetricsRecorder(openTelemetry, BuiltInMetricsConstant.METER_NAME),
102+
attributes,
103+
new TraceWrapper(
104+
Tracing.getTracer(),
105+
openTelemetry.getTracer(
106+
MetricRegistryConstants.INSTRUMENTATION_SCOPE,
107+
GaxProperties.getLibraryVersion(getClass())),
108+
true));
100109
}
101110

102111
@BeforeClass
@@ -115,6 +124,12 @@ public void clearRequests() throws IOException {
115124
@Override
116125
public void createSpannerInstance() {
117126
SpannerOptions.Builder builder = SpannerOptions.newBuilder();
127+
128+
ApiTracerFactory metricsTracerFactory =
129+
new BuiltInMetricsTracerFactory(
130+
new BuiltInMetricsRecorder(OpenTelemetry.noop(), BuiltInMetricsConstant.METER_NAME),
131+
attributes,
132+
new TraceWrapper(Tracing.getTracer(), OpenTelemetry.noop().getTracer(""), true));
118133
// Set a quick polling algorithm to prevent this from slowing down the test unnecessarily.
119134
builder
120135
.getDatabaseAdminStubSettingsBuilder()

0 commit comments

Comments
 (0)