Skip to content
This repository was archived by the owner on Aug 2, 2022. It is now read-only.

Commit 52e5cc4

Browse files
mitalawachatmitalawachat
authored
Track admission control metrics (#290)
* Track admission control metrics Co-authored-by: Mital Awachat <[email protected]>
1 parent c280915 commit 52e5cc4

File tree

4 files changed

+234
-0
lines changed

4 files changed

+234
-0
lines changed

‎src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerPlugin.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import static java.util.Collections.singletonList;
1919

2020
import com.amazon.opendistro.elasticsearch.performanceanalyzer.action.PerformanceAnalyzerActionFilter;
21+
import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.AdmissionControlMetricsCollector;
2122
import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.CacheConfigMetricsCollector;
2223
import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.CircuitBreakerCollector;
2324
import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.DisksCollector;
@@ -203,6 +204,7 @@ public PerformanceAnalyzerPlugin(final Settings settings, final java.nio.file.Pa
203204
performanceAnalyzerController,configOverridesWrapper));
204205
scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new MasterThrottlingMetricsCollector(
205206
performanceAnalyzerController,configOverridesWrapper));
207+
scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new AdmissionControlMetricsCollector());
206208
try {
207209
Class.forName(ShardIndexingPressureMetricsCollector.SHARD_INDEXING_PRESSURE_CLASS_NAME);
208210
scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new ShardIndexingPressureMetricsCollector(

‎src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/AdmissionControlMetricsCollector.java‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
/*
2+
* Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors;
17+
18+
import com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerApp;
19+
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics;
20+
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration;
21+
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsProcessor;
22+
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
23+
import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.metrics.ExceptionsAndErrors;
24+
import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.metrics.WriterMetrics;
25+
import com.fasterxml.jackson.annotation.JsonProperty;
26+
import org.apache.logging.log4j.LogManager;
27+
import org.apache.logging.log4j.Logger;
28+
29+
import java.lang.reflect.Method;
30+
import java.util.Objects;
31+
32+
/**
33+
* AdmissionControlMetricsCollector collects `UsedQuota`, `TotalQuota`, RejectionCount
34+
*/
35+
public class AdmissionControlMetricsCollector extends PerformanceAnalyzerMetricsCollector implements MetricsProcessor {
36+
37+
private static final Logger LOG = LogManager.getLogger(AdmissionControlMetricsCollector.class);
38+
private static final int sTimeInterval = MetricsConfiguration.SAMPLING_INTERVAL;
39+
private static final int KEYS_PATH_LENGTH = 0;
40+
private StringBuilder value;
41+
42+
// Global JVM Memory Pressure Controller
43+
private final static String GLOBAL_JVMMP = "Global_JVMMP";
44+
45+
// Request Size Controller
46+
private final static String REQUEST_SIZE = "Request_Size";
47+
48+
private final static String ADMISSION_CONTROLLER =
49+
"com.sonian.elasticsearch.http.jetty.throttling.AdmissionController";
50+
51+
private final static String ADMISSION_CONTROL_SERVICE =
52+
"com.sonian.elasticsearch.http.jetty.throttling.JettyAdmissionControlService";
53+
54+
public AdmissionControlMetricsCollector() {
55+
super(sTimeInterval, "AdmissionControlMetricsCollector");
56+
this.value = new StringBuilder();
57+
}
58+
59+
@Override
60+
@SuppressWarnings("unchecked")
61+
public void collectMetrics(long startTime) {
62+
if(!isAdmissionControlFeatureAvailable()) {
63+
LOG.debug("AdmissionControl is not available for this domain");
64+
PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat(
65+
WriterMetrics.ADMISSION_CONTROL_COLLECTOR_NOT_AVAILABLE, "", 1);
66+
return;
67+
}
68+
69+
long startTimeMillis = System.currentTimeMillis();
70+
try {
71+
Class admissionController = Class.forName(ADMISSION_CONTROLLER);
72+
Class jettyAdmissionControlService = Class.forName(ADMISSION_CONTROL_SERVICE);
73+
74+
Method getAdmissionController = jettyAdmissionControlService.getDeclaredMethod(
75+
"getAdmissionController",
76+
String.class);
77+
78+
Object globalJVMMP = getAdmissionController.invoke(null, GLOBAL_JVMMP);
79+
Object requestSize = getAdmissionController.invoke(null, REQUEST_SIZE);
80+
81+
if(Objects.isNull(globalJVMMP) && Objects.isNull(requestSize)) {
82+
return;
83+
}
84+
85+
value.setLength(0);
86+
87+
Method getUsedQuota = admissionController.getDeclaredMethod("getUsedQuota");
88+
Method getTotalQuota = admissionController.getDeclaredMethod("getTotalQuota");
89+
Method getRejectionCount = admissionController.getDeclaredMethod("getRejectionCount");
90+
91+
if(!Objects.isNull(globalJVMMP)) {
92+
value.append(PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds())
93+
.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor)
94+
.append(new AdmissionControlMetrics(
95+
GLOBAL_JVMMP,
96+
(long)getUsedQuota.invoke(globalJVMMP),
97+
(long)getTotalQuota.invoke(globalJVMMP),
98+
(long)getRejectionCount.invoke(globalJVMMP)
99+
).serialize());
100+
}
101+
102+
if(!Objects.isNull(requestSize)) {
103+
value.append(PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds())
104+
.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor)
105+
.append(new AdmissionControlMetrics(
106+
REQUEST_SIZE,
107+
(long)getUsedQuota.invoke(requestSize),
108+
(long)getTotalQuota.invoke(requestSize),
109+
(long)getRejectionCount.invoke(requestSize)
110+
).serialize());
111+
}
112+
113+
saveMetricValues(value.toString(), startTime);
114+
115+
PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat(
116+
WriterMetrics.ADMISSION_CONTROL_COLLECTOR_EXECUTION_TIME, "",
117+
System.currentTimeMillis() - startTimeMillis);
118+
119+
} catch(Exception ex) {
120+
PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
121+
ExceptionsAndErrors.ADMISSION_CONTROL_COLLECTOR_ERROR, getCollectorName(),
122+
System.currentTimeMillis() - startTimeMillis);
123+
LOG.debug("Exception in collecting AdmissionControl Metrics: {} for startTime {}",
124+
ex::toString, () -> startTime);
125+
}
126+
}
127+
128+
@Override
129+
public String getMetricsPath(long startTime, String... keysPath) {
130+
if (keysPath.length != KEYS_PATH_LENGTH) {
131+
throw new RuntimeException("keys length should be " + KEYS_PATH_LENGTH);
132+
}
133+
return PerformanceAnalyzerMetrics.generatePath(startTime,
134+
PerformanceAnalyzerMetrics.sAdmissionControlMetricsPath);
135+
}
136+
137+
static class AdmissionControlMetrics extends MetricStatus {
138+
139+
private String controllerName;
140+
private long current;
141+
private long threshold;
142+
private long rejectionCount;
143+
144+
public AdmissionControlMetrics() {
145+
super();
146+
}
147+
148+
public AdmissionControlMetrics(String controllerName, long current, long threshold, long rejectionCount) {
149+
super();
150+
this.controllerName = controllerName;
151+
this.current = current;
152+
this.threshold = threshold;
153+
this.rejectionCount = rejectionCount;
154+
}
155+
156+
@JsonProperty(AllMetrics.AdmissionControlDimension.Constants.CONTROLLER_NAME)
157+
public String getControllerName() {
158+
return controllerName;
159+
}
160+
161+
@JsonProperty(AllMetrics.AdmissionControlValue.Constants.CURRENT_VALUE)
162+
public long getCurrent() {
163+
return current;
164+
}
165+
166+
@JsonProperty(AllMetrics.AdmissionControlValue.Constants.THRESHOLD_VALUE)
167+
public long getThreshold() {
168+
return threshold;
169+
}
170+
171+
@JsonProperty(AllMetrics.AdmissionControlValue.Constants.REJECTION_COUNT)
172+
public long getRejectionCount() {
173+
return rejectionCount;
174+
}
175+
}
176+
177+
private boolean isAdmissionControlFeatureAvailable() {
178+
try {
179+
Class.forName(ADMISSION_CONTROLLER);
180+
Class.forName(ADMISSION_CONTROL_SERVICE);
181+
} catch (ClassNotFoundException e) {
182+
return false;
183+
}
184+
return true;
185+
}
186+
}

‎src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/util/Utils.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
package com.amazon.opendistro.elasticsearch.performanceanalyzer.util;
1717

1818
import com.amazon.opendistro.elasticsearch.performanceanalyzer.ESResources;
19+
import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.AdmissionControlMetricsCollector;
1920
import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.CacheConfigMetricsCollector;
2021
import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.FaultDetectionMetricsCollector;
2122
import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.ShardIndexingPressureMetricsCollector;
@@ -47,6 +48,7 @@ public class Utils {
4748

4849
public static void configureMetrics() {
4950
MetricsConfiguration.MetricConfig cdefault = MetricsConfiguration.cdefault ;
51+
MetricsConfiguration.CONFIG_MAP.put(AdmissionControlMetricsCollector.class, cdefault);
5052
MetricsConfiguration.CONFIG_MAP.put(CacheConfigMetricsCollector.class, cdefault);
5153
MetricsConfiguration.CONFIG_MAP.put(CircuitBreakerCollector.class, cdefault);
5254
MetricsConfiguration.CONFIG_MAP.put(ThreadPoolMetricsCollector.class, cdefault);

‎src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/collectors/AdmissionControlMetricsCollectorTests.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
* Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors;
17+
18+
import com.amazon.opendistro.elasticsearch.performanceanalyzer.CustomMetricsLocationTestBase;
19+
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.MetricsConfiguration;
20+
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
21+
import com.amazon.opendistro.elasticsearch.performanceanalyzer.reader_writer_shared.Event;
22+
import org.junit.Test;
23+
24+
import java.util.ArrayList;
25+
import java.util.List;
26+
27+
import static org.junit.Assert.assertEquals;
28+
29+
public class AdmissionControlMetricsCollectorTests extends CustomMetricsLocationTestBase {
30+
31+
@Test
32+
public void admissionControlMetricsCollector() {
33+
MetricsConfiguration.CONFIG_MAP.put(AdmissionControlMetricsCollector.class, MetricsConfiguration.cdefault);
34+
AdmissionControlMetricsCollector admissionControlMetricsCollector = new AdmissionControlMetricsCollector();
35+
36+
long startTimeInMills = System.currentTimeMillis();
37+
admissionControlMetricsCollector.saveMetricValues("testMetric", startTimeInMills);
38+
39+
List<Event> metrics = new ArrayList<>();
40+
PerformanceAnalyzerMetrics.metricQueue.drainTo(metrics);
41+
assertEquals(1, metrics.size());
42+
assertEquals("testMetric", metrics.get(0).value);
43+
}
44+
}

0 commit comments

Comments
 (0)