Skip to content

Commit 8f914a4

Browse files
eirsepeirsep
committed
Enables/disables thread contention monitoring based on setting
Signed-off-by: Surya Sashank Nistala <[email protected]>
1 parent bbc6813 commit 8f914a4

File tree

9 files changed

+291
-6
lines changed

9 files changed

+291
-6
lines changed

‎null/data/batch_metrics_enabled.conf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
true

‎null/data/thread_contention_monitoring_enabled.conf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
true

‎src/main/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerApp.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
import org.opensearch.performanceanalyzer.rca.samplers.BatchMetricsEnabledSampler;
6868
import org.opensearch.performanceanalyzer.rca.samplers.MetricsDBFileSampler;
6969
import org.opensearch.performanceanalyzer.rca.samplers.RcaStateSamplers;
70+
import org.opensearch.performanceanalyzer.rca.samplers.ThreadContentionMonitoringEnabledSampler;
7071
import org.opensearch.performanceanalyzer.rca.stats.RcaStatsReporter;
7172
import org.opensearch.performanceanalyzer.rca.stats.collectors.SampleAggregator;
7273
import org.opensearch.performanceanalyzer.rca.stats.emitters.ISampler;
@@ -364,6 +365,7 @@ public static List<ISampler> getAllSamplers(final AppContext appContext) {
364365
allSamplers.add(RcaStateSamplers.getRcaEnabledSampler(appContext));
365366
allSamplers.add(new BatchMetricsEnabledSampler(appContext));
366367
allSamplers.add(new MetricsDBFileSampler(appContext));
368+
allSamplers.add(new ThreadContentionMonitoringEnabledSampler(appContext));
367369

368370
return allSamplers;
369371
}
@@ -373,6 +375,7 @@ private static MeasurementSet[] getPeriodicMeasurementSets() {
373375
measurementSets.addAll(Arrays.asList(JvmMetrics.values()));
374376
measurementSets.add(RcaRuntimeMetrics.RCA_ENABLED);
375377
measurementSets.add(ReaderMetrics.BATCH_METRICS_ENABLED);
378+
measurementSets.add(ReaderMetrics.THREAD_CONTENTION_MONITORING_ENABLED);
376379
measurementSets.add(ReaderMetrics.METRICSDB_NUM_FILES);
377380
measurementSets.add(ReaderMetrics.METRICSDB_SIZE_FILES);
378381
measurementSets.add(ReaderMetrics.METRICSDB_NUM_UNCOMPRESSED_FILES);

‎src/main/java/org/opensearch/performanceanalyzer/jvm/ThreadList.java‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
import org.opensearch.performanceanalyzer.core.Util;
5252
import org.opensearch.performanceanalyzer.metrics.MetricsConfiguration;
5353
import org.opensearch.performanceanalyzer.rca.framework.metrics.WriterMetrics;
54+
import org.opensearch.performanceanalyzer.reader.ReaderMetricsProcessor;
5455
import sun.tools.attach.HotSpotVirtualMachine;
5556

5657
/** Traverses and prints the stack traces for all Java threads in the remote VM */
@@ -73,12 +74,6 @@ public class ThreadList {
7374

7475
private static Lock vmAttachLock = new ReentrantLock();
7576

76-
static {
77-
if (threadBean.isThreadContentionMonitoringSupported()) {
78-
threadBean.setThreadContentionMonitoringEnabled(true);
79-
}
80-
}
81-
8277
public static class ThreadState {
8378
public long javaTid;
8479
public long nativeTid;
@@ -146,6 +141,13 @@ public String toString() {
146141
* @return A hashmap of threadId to threadState.
147142
*/
148143
public static Map<Long, ThreadState> getNativeTidMap() {
144+
if (threadBean.isThreadContentionMonitoringSupported()) {
145+
if (ReaderMetricsProcessor.getInstance().getThreadContentionMonitoringEnabled()) {
146+
threadBean.setThreadContentionMonitoringEnabled(true);
147+
} else {
148+
threadBean.setThreadContentionMonitoringEnabled(false);
149+
}
150+
}
149151
if (vmAttachLock.tryLock()) {
150152
try {
151153
// Thread dumps are expensive and therefore we make sure that at least

‎src/main/java/org/opensearch/performanceanalyzer/rca/framework/metrics/ReaderMetrics.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,10 @@ public enum ReaderMetrics implements MeasurementSet {
124124
"millis",
125125
Arrays.asList(Statistics.MAX, Statistics.MEAN, Statistics.SUM)),
126126

127+
/** Whether or not thread contention monitoring is enabled (0 for enabled, 1 for disabled). */
128+
THREAD_CONTENTION_MONITORING_ENABLED(
129+
"ThreadContentionMonitoringEnabled", "count", Statistics.SAMPLE),
130+
127131
/** Amount of time taken to emit Shard State metrics. */
128132
SHARD_STATE_EMITTER_EXECUTION_TIME(
129133
"ShardStateEmitterExecutionTime",

‎src/main/java/org/opensearch/performanceanalyzer/rca/samplers/ThreadContentionMonitoringEnabledSampler.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*
8+
* Modifications Copyright OpenSearch Contributors. See
9+
* GitHub history for details.
10+
*/
11+
12+
/*
13+
* Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
14+
*
15+
* Licensed under the Apache License, Version 2.0 (the "License").
16+
* You may not use this file except in compliance with the License.
17+
* A copy of the License is located at
18+
*
19+
* http://www.apache.org/licenses/LICENSE-2.0
20+
*
21+
* or in the "license" file accompanying this file. This file is distributed
22+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
23+
* express or implied. See the License for the specific language governing
24+
* permissions and limitations under the License.
25+
*/
26+
27+
package org.opensearch.performanceanalyzer.rca.samplers;
28+
29+
30+
import java.util.Objects;
31+
import org.opensearch.performanceanalyzer.AppContext;
32+
import org.opensearch.performanceanalyzer.rca.framework.metrics.ReaderMetrics;
33+
import org.opensearch.performanceanalyzer.rca.framework.util.InstanceDetails;
34+
import org.opensearch.performanceanalyzer.rca.stats.collectors.SampleAggregator;
35+
import org.opensearch.performanceanalyzer.rca.stats.emitters.ISampler;
36+
import org.opensearch.performanceanalyzer.reader.ReaderMetricsProcessor;
37+
38+
public class ThreadContentionMonitoringEnabledSampler implements ISampler {
39+
private final AppContext appContext;
40+
41+
public ThreadContentionMonitoringEnabledSampler(final AppContext appContext) {
42+
Objects.requireNonNull(appContext);
43+
this.appContext = appContext;
44+
}
45+
46+
@Override
47+
public void sample(SampleAggregator sampleCollector) {
48+
sampleCollector.updateStat(
49+
ReaderMetrics.THREAD_CONTENTION_MONITORING_ENABLED,
50+
"",
51+
isThreadContentionMonitoringEnabled() ? 1 : 0);
52+
}
53+
54+
boolean isThreadContentionMonitoringEnabled() {
55+
return ReaderMetricsProcessor.getInstance().getThreadContentionMonitoringEnabled();
56+
}
57+
}

‎src/main/java/org/opensearch/performanceanalyzer/reader/ReaderMetricsProcessor.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,8 +103,12 @@ public class ReaderMetricsProcessor implements Runnable {
103103
private final ConfigOverridesApplier configOverridesApplier;
104104

105105
public static final String BATCH_METRICS_ENABLED_CONF_FILE = "batch_metrics_enabled.conf";
106+
public static final String THREAD_CONTENTION_MONITORING_ENABLED_CONF_FILE =
107+
"thread_contention_monitoring_enabled.conf";
106108
private boolean batchMetricsEnabled;
107109
public static final boolean defaultBatchMetricsEnabled = false;
110+
private boolean threadContentionMonitoringEnabled;
111+
public static final boolean defaultThreadContentionMonitoringEnabled = false;
108112
// This needs to be concurrent since it may be concurrently accessed by the metrics processor
109113
// thread and the query handler thread.
110114
private ConcurrentSkipListSet<Long> batchMetricsDBSet;
@@ -156,6 +160,7 @@ public ReaderMetricsProcessor(
156160
batchMetricsDBSet = new ConcurrentSkipListSet<>();
157161
readBatchMetricsEnabledFromConf();
158162
restoreBatchMetricsState();
163+
threadContentionMonitoringEnabled = defaultBatchMetricsEnabled;
159164
}
160165

161166
@Override
@@ -1011,6 +1016,34 @@ public boolean getBatchMetricsEnabled() {
10111016
return batchMetricsEnabled;
10121017
}
10131018

1019+
private void readThreadContentionMonitoringEnabledFromConf() {
1020+
Path filePath = Paths.get(Util.DATA_DIR, THREAD_CONTENTION_MONITORING_ENABLED_CONF_FILE);
1021+
1022+
Util.invokePrivileged(
1023+
() -> {
1024+
try (Scanner sc = new Scanner(filePath)) {
1025+
String nextLine = sc.nextLine();
1026+
boolean oldValue = threadContentionMonitoringEnabled;
1027+
boolean newValue = Boolean.parseBoolean(nextLine);
1028+
if (oldValue != newValue) {
1029+
threadContentionMonitoringEnabled = newValue;
1030+
LOG.info(
1031+
"Thread contention monitoring enabled changed from {} to {}",
1032+
oldValue,
1033+
newValue);
1034+
}
1035+
} catch (IOException e) {
1036+
LOG.error("Error reading file '{}': {}", filePath.toString(), e);
1037+
threadContentionMonitoringEnabled =
1038+
defaultThreadContentionMonitoringEnabled;
1039+
}
1040+
});
1041+
}
1042+
1043+
public boolean getThreadContentionMonitoringEnabled() {
1044+
return threadContentionMonitoringEnabled;
1045+
}
1046+
10141047
/**
10151048
* An example value is this: current_time:1566413987194 StartTime:1566413987194 ItemCount:359
10161049
* IndexName:nyc_taxis ShardID:25 Primary:true Each pair is separated by new line and the key
@@ -1089,4 +1122,9 @@ NavigableMap<Long, MetricsDB> getMetricsDBMap() {
10891122
public void readBatchMetricsEnabledFromConfShim() {
10901123
readBatchMetricsEnabledFromConf();
10911124
}
1125+
1126+
@VisibleForTesting
1127+
public void readThreadContentionMonitoringEnabledFromConfShim() {
1128+
readThreadContentionMonitoringEnabledFromConf();
1129+
}
10921130
}

‎src/test/java/org/opensearch/performanceanalyzer/rca/samplers/ThreadContentionMonitoringEnabledSamplerTest.java‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*
8+
* Modifications Copyright OpenSearch Contributors. See
9+
* GitHub history for details.
10+
*/
11+
12+
/*
13+
* Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
14+
*
15+
* Licensed under the Apache License, Version 2.0 (the "License").
16+
* You may not use this file except in compliance with the License.
17+
* A copy of the License is located at
18+
*
19+
* http://www.apache.org/licenses/LICENSE-2.0
20+
*
21+
* or in the "license" file accompanying this file. This file is distributed
22+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
23+
* express or implied. See the License for the specific language governing
24+
* permissions and limitations under the License.
25+
*/
26+
27+
package org.opensearch.performanceanalyzer.rca.samplers;
28+
29+
import static org.junit.Assert.assertFalse;
30+
import static org.junit.Assert.assertTrue;
31+
import static org.mockito.Mockito.times;
32+
import static org.mockito.Mockito.verify;
33+
34+
import java.io.IOException;
35+
import java.nio.file.Files;
36+
import java.nio.file.Path;
37+
import java.nio.file.Paths;
38+
import java.util.Collections;
39+
import org.junit.Before;
40+
import org.junit.BeforeClass;
41+
import org.junit.Test;
42+
import org.mockito.Mock;
43+
import org.mockito.MockitoAnnotations;
44+
import org.opensearch.performanceanalyzer.AppContext;
45+
import org.opensearch.performanceanalyzer.core.Util;
46+
import org.opensearch.performanceanalyzer.rca.framework.metrics.ReaderMetrics;
47+
import org.opensearch.performanceanalyzer.rca.stats.collectors.SampleAggregator;
48+
import org.opensearch.performanceanalyzer.reader.ClusterDetailsEventProcessor;
49+
import org.opensearch.performanceanalyzer.reader.ClusterDetailsEventProcessorTestHelper;
50+
import org.opensearch.performanceanalyzer.reader.ReaderMetricsProcessor;
51+
52+
public class ThreadContentionMonitoringEnabledSamplerTest {
53+
private static Path threadContentionMonitoringEnabledConfFile;
54+
private static String rootLocation;
55+
private static ReaderMetricsProcessor mp;
56+
private static AppContext appContext;
57+
private static ThreadContentionMonitoringEnabledSampler uut;
58+
59+
@Mock private SampleAggregator sampleAggregator;
60+
61+
@BeforeClass
62+
public static void setUpClass() throws Exception {
63+
Files.createDirectories(Paths.get(Util.DATA_DIR));
64+
threadContentionMonitoringEnabledConfFile =
65+
Paths.get(
66+
Util.DATA_DIR,
67+
ReaderMetricsProcessor.THREAD_CONTENTION_MONITORING_ENABLED_CONF_FILE);
68+
Files.deleteIfExists(threadContentionMonitoringEnabledConfFile);
69+
70+
rootLocation = "build/resources/test/reader/";
71+
mp = new ReaderMetricsProcessor(rootLocation);
72+
ReaderMetricsProcessor.setCurrentInstance(mp);
73+
74+
appContext = new AppContext();
75+
uut = new ThreadContentionMonitoringEnabledSampler(appContext);
76+
}
77+
78+
@Before
79+
public void setUp() {
80+
MockitoAnnotations.initMocks(this);
81+
}
82+
83+
private void writeThreadContentionMonitoringEnabled(boolean enabled) throws IOException {
84+
Files.write(
85+
threadContentionMonitoringEnabledConfFile, Boolean.toString(enabled).getBytes());
86+
}
87+
88+
private void clearThreadContentionMonitoringEnabled() throws IOException {
89+
Files.deleteIfExists(threadContentionMonitoringEnabledConfFile);
90+
}
91+
92+
@Test
93+
public void testIsThreadContentionMonitoringEnabled_notMaster() {
94+
appContext.setClusterDetailsEventProcessor(null);
95+
assertFalse(uut.isThreadContentionMonitoringEnabled());
96+
}
97+
98+
@Test
99+
public void testIsThreadContentionMonitoringEnabled() throws IOException {
100+
ClusterDetailsEventProcessor clusterDetailsEventProcessor =
101+
new ClusterDetailsEventProcessor();
102+
ClusterDetailsEventProcessor.NodeDetails details =
103+
ClusterDetailsEventProcessorTestHelper.newNodeDetails("nodex", "127.0.0.1", true);
104+
clusterDetailsEventProcessor.setNodesDetails(Collections.singletonList(details));
105+
appContext.setClusterDetailsEventProcessor(clusterDetailsEventProcessor);
106+
107+
// No thread contention monitoring enabled file
108+
clearThreadContentionMonitoringEnabled();
109+
mp.readThreadContentionMonitoringEnabledFromConfShim();
110+
assertFalse(uut.isThreadContentionMonitoringEnabled());
111+
112+
// thread contention monitoring disabled
113+
writeThreadContentionMonitoringEnabled(false);
114+
mp.readThreadContentionMonitoringEnabledFromConfShim();
115+
assertFalse(uut.isThreadContentionMonitoringEnabled());
116+
117+
// thread contention monitoring disabled
118+
writeThreadContentionMonitoringEnabled(true);
119+
mp.readThreadContentionMonitoringEnabledFromConfShim();
120+
assertTrue(uut.isThreadContentionMonitoringEnabled());
121+
}
122+
123+
@Test
124+
public void testSample() {
125+
ClusterDetailsEventProcessor clusterDetailsEventProcessor =
126+
new ClusterDetailsEventProcessor();
127+
ClusterDetailsEventProcessor.NodeDetails details =
128+
ClusterDetailsEventProcessorTestHelper.newNodeDetails("nodex", "127.0.0.1", true);
129+
clusterDetailsEventProcessor.setNodesDetails(Collections.singletonList(details));
130+
appContext.setClusterDetailsEventProcessor(clusterDetailsEventProcessor);
131+
132+
uut.sample(sampleAggregator);
133+
verify(sampleAggregator, times(1))
134+
.updateStat(
135+
ReaderMetrics.THREAD_CONTENTION_MONITORING_ENABLED,
136+
"",
137+
mp.getThreadContentionMonitoringEnabled() ? 1 : 0);
138+
}
139+
}

‎src/test/java/org/opensearch/performanceanalyzer/reader/ReaderMetricsProcessorTests.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import static org.junit.Assert.assertTrue;
3333
import static org.junit.Assert.fail;
3434
import static org.opensearch.performanceanalyzer.reader.ReaderMetricsProcessor.BATCH_METRICS_ENABLED_CONF_FILE;
35+
import static org.opensearch.performanceanalyzer.reader.ReaderMetricsProcessor.THREAD_CONTENTION_MONITORING_ENABLED_CONF_FILE;
3536

3637
import java.io.File;
3738
import java.io.FilenameFilter;
@@ -409,6 +410,45 @@ public void testReadBatchMetricsEnabledFromConf() throws Exception {
409410
mp.getBatchMetricsEnabled() == ReaderMetricsProcessor.defaultBatchMetricsEnabled);
410411
}
411412

413+
@Test
414+
public void testReadThreadContentionMonitoringEnabledFromConf() throws Exception {
415+
Files.createDirectories(Paths.get(Util.DATA_DIR));
416+
Path threadContentionMonitoringEnabledConfFile =
417+
Paths.get(Util.DATA_DIR, THREAD_CONTENTION_MONITORING_ENABLED_CONF_FILE);
418+
Files.deleteIfExists(threadContentionMonitoringEnabledConfFile);
419+
ReaderMetricsProcessor mp = new ReaderMetricsProcessor(rootLocation);
420+
ReaderMetricsProcessor.setCurrentInstance(mp);
421+
422+
// Test default
423+
assertTrue(
424+
mp.getThreadContentionMonitoringEnabled()
425+
== ReaderMetricsProcessor.defaultThreadContentionMonitoringEnabled);
426+
427+
// Test disabled
428+
Files.write(threadContentionMonitoringEnabledConfFile, Boolean.toString(false).getBytes());
429+
mp.readThreadContentionMonitoringEnabledFromConfShim();
430+
assertFalse(mp.getThreadContentionMonitoringEnabled());
431+
432+
// Test reverts back to default when file is deleted
433+
Files.delete(threadContentionMonitoringEnabledConfFile);
434+
mp.readThreadContentionMonitoringEnabledFromConfShim();
435+
assertTrue(
436+
mp.getThreadContentionMonitoringEnabled()
437+
== ReaderMetricsProcessor.defaultThreadContentionMonitoringEnabled);
438+
439+
// Test enabled
440+
Files.write(threadContentionMonitoringEnabledConfFile, Boolean.toString(true).getBytes());
441+
mp.readThreadContentionMonitoringEnabledFromConfShim();
442+
assertTrue(mp.getThreadContentionMonitoringEnabled());
443+
444+
// Test reverts back to default when file is deleted
445+
Files.delete(threadContentionMonitoringEnabledConfFile);
446+
mp.readThreadContentionMonitoringEnabledFromConfShim();
447+
assertTrue(
448+
mp.getThreadContentionMonitoringEnabled()
449+
== ReaderMetricsProcessor.defaultThreadContentionMonitoringEnabled);
450+
}
451+
412452
@Test
413453
public void testGetBatchMetrics() throws Exception {
414454
deleteAll();

0 commit comments

Comments
 (0)