Merge branch 'add_idle_test' of github.com:tanvikini/openj9-systemtest into add_idle_test

Signed-off-by: Tanvi Kini <[email protected]>

Author: tanvikini

openj9.build/makefile

@@ -416,11 +416,18 @@ test.SharedClasses.SCM23.MultiThread \
 test.SharedClasses.SCM23.MultiThreadMultiCL
 SHARED_CLASSES_TESTS:=$(filter-out $(EXCLUDE),$(SHARED_CLASSES_TESTS))
 
+IDLE_TESTS:=test.IdleBenchMark_MinIdleWaitTime \
+test.IdleBenchMark_GcOnIdle \
+test.IdleBenchMark_CompactOnIdle
+IDLE_TESTS:=$(filter-out $(EXCLUDE),$(IDLE_TESTS))
+
 TEST_TARGETS:=test.list \
 test.help \
 $(DAA_TESTS) \
 $(GC_TESTS) \
-$(SHARED_CLASSES_TESTS)
+$(SHARED_CLASSES_TESTS) \
+$(IDLE_TESTS)
+
 TEST_TARGETS:=$(filter-out $(EXCLUDE),$(TEST_TARGETS))
 
 .PHONY: build configure clean refresh_source test $(TEST_TARGETS)
@@ -447,6 +454,7 @@ test: $(TEST_TARGETS)
 test.daa: $(DAA_TESTS)
 test.gc: $(GC_TESTS)
 test.SharedClasses: $(SHARED_CLASSES_TESTS)
+test.idle: $(IDLE_TESTS)
 
 test.list:
 	echo Running target $@
@@ -534,15 +542,15 @@ test.SharedClasses.SCM23.MultiThreadMultiCL:
 	echo Target $@ completed
 test.IdleBenchMark_MinIdleWaitTime:
 	echo Running target $@
-	$(STF_COMMAND) -test=IdleLoadTest -java-args="-XX:IdleTuningMinIdleWaitTime=180 -Xmx1024m -Xjit:verbose={compilePerformance},vlog=jitlog" $(LOG)
+	$(STF_COMMAND) -test=IdleLoadTest -test-args="variation=MinIdleWaitTime" $(LOG)
 	echo Target $@ completed
 test.IdleBenchMark_GcOnIdle:
 	echo Running target $@
-	$(STF_COMMAND) -test=IdleLoadTest -java-args="-XX:+IdleTuningGcOnIdle -Xtune:virtualized -XX:IdleTuningMinIdleWaitTime=120 -Xmx1024m -verbose:gc -Xverbosegclog:gc.verbose -Xjit:verbose={compilePerformance},vlog=jitlog" $(LOG)
+	$(STF_COMMAND) -test=IdleLoadTest -test-args="variation=GcOnIdle" $(LOG)
 	echo Target $@ completed
 test.IdleBenchMark_CompactOnIdle:
 	echo Running target $@
-	$(STF_COMMAND) -test=IdleLoadTest -java-args="-XX:+IdleTuningCompactOnIdle -Xtune:virtualized -XX:IdleTuningMinIdleWaitTime=120 -Xmx1024m -verbose:gc -Xverbosegclog:gc.verbose -Xjit:verbose={compilePerformance},vlog=jitlog" $(LOG)
+	$(STF_COMMAND) -test=IdleLoadTest -test-args="variation=CompactOnIdle" $(LOG)
 	echo Target $@ completed 
 
 help:

openj9.test.idle/docs/README.md

@@ -0,0 +1,29 @@
+openj9.test.idle
+- This project contains a stress version of the Idle Micro Benchmark test which aims to test the Idle detection and management feature available in J9 vm. For more information on the feature and related -XX JVM options, visit : https://www.ibm.com/support/knowledgecenter/SSYKE2_8.0.0/com.ibm.java.lnx.80.doc/diag/appendixes/cmdline/commands_jvm_xx.html.
+- The test has been written to have alternating active and idle cycles.
+- Active cycles are periods of time when the test will perform CPU and Memory Intensive operations to consume CPU cycles and Heap Memory.
+- Idle cycles are longer periods of time where the test sleeps/ performs no activity.
+- Idle detection and management are enabled/disabled via newly added JVM options. 
+- The Idle detection feature is available on all platforms whereas Idle management feature is supported only on Linux x86 platforms.
+- The test exercises the newly added JVM options and checks if they behave as expected under stress conditions.
+
+- There are 3 variations to this test 
+
+1. MinIdleWaitTime 
+- Tests the -XX:IdleTuningMinIdleWaitTime option. This is the minimum amount of time the application needs to be idle to be detected as Idle.
+- Applicable on all platforms.
+- Uses the following Java arguments while running the test 
+	-XX:IdleTuningMinIdleWaitTime=180 -Xmx1024m -Xjit:verbose={compilePerformance},vlog=jitlog
+
+2. GcOnIdle
+- Tests the -XX:+IdleTuningGcOnIdle and when enabled, GC kicks in during the Idle cycle and reduces the footprint of the application. 
+- Applicable only on Linux_x86-32 and Linux_x86-64 platforms as of now.
+- Uses the following JVM arguments while running the test 
+	-XX:+IdleTuningGcOnIdle -Xtune:virtualized -XX:IdleTuningMinIdleWaitTime=120 -Xmx1024m -verbose:gc -Xverbosegclog:gc.verbose -Xjit:verbose={compilePerformance},vlog=jitlog" $(LOG)
+
+3. CompactOnIdle 
+- Tests the -XX:+IdleTuningCompactOnIdle and when enabled, GC performs compaction on the heap during idle cycle to create contiguious free memory spaces for newer allocations.
+- Applicable only on Linux_x86-32 and Linux_x86-64 platforms as of now.
+- Uses the following JVM arguments while running the test
+	-XX:+IdleTuningCompactOnIdle -Xtune:virtualized -XX:IdleTuningMinIdleWaitTime=120 -Xmx1024m -verbose:gc -Xverbosegclog:gc.verbose -Xjit:verbose={compilePerformance},vlog=jitlog
+

openj9.test.idle/src/test.idle/net/openj9/test/IdleMicroBenchmark.java

@@ -116,15 +116,15 @@ public static void main(String args[]) throws MalformedObjectNameException, IOEx
 				if (optionValue.equals("transactional") || optionValue.equals("image") || optionValue.equals("spike")) {
 					workloadType = optionValue;
 					if (optionValue.equals("transactional")) {
-						nThreads = 15;
+						nThreads = 8;
 						workloadTypeNum = 1;
 					}
 					if (optionValue.equals("image")) {
-						nThreads = 10;
+						nThreads = 5;
 						workloadTypeNum = 3;
 					}
 					if (optionValue.equals("spike")) {
-						nThreads = 5;
+						nThreads = 3;
 						workloadTypeNum = 0;
 					}
 				} else {
@@ -167,6 +167,7 @@ public static void main(String args[]) throws MalformedObjectNameException, IOEx
 		try {
 			IdleMicroBenchmark imbm1 = new IdleMicroBenchmark();
 			long memoryLimitInBytes = memoryLimit * 1024 * 1024;
+			//multiply sleepTime by 1000L to convert seconds to milliseconds as it is passed to Thread.sleep();
 			sleepTime = sleepTime * 1000L;
 			System.setProperty("isIdle", "false");
 
@@ -219,7 +220,7 @@ public static void main(String args[]) throws MalformedObjectNameException, IOEx
 	void induceCombinedLoad(int threads, int interval, int workloadTypeNum, long runFor, long  memoryLimit, ExecutorService executor) {
 
 		List<Future<Long>> combinedWorkerList = new ArrayList<Future<Long>>();
-		long stopRunning = System.currentTimeMillis() + 1000L * runFor;
+		long stopRunning = System.currentTimeMillis() + 1000L * runFor; 
 		java.lang.management.MemoryMXBean membean = ManagementFactory.getMemoryMXBean();
 		long heapInBytes = 0;
 		int iter = 1;
@@ -264,12 +265,6 @@ void induceCombinedLoad(int threads, int interval, int workloadTypeNum, long run
 
 				System.out.println("CombinedLoad Load: Iteration: " + iter + ": Complete. Time: " + timeDiff + " Heap: " + heapInBytes);
 
-				try {
-					//Thread.sleep(10000);
-				} catch (Exception e) {
-					e.printStackTrace();
-					System.exit(1);
-				}
 			}
 
 			iter++;

openj9.test.load/config/inventories/idle/idleTest.xml

@@ -1,5 +1,5 @@
 <inventory> 
 	<arbitraryJava class="net.openj9.test.IdleMicroBenchmark" method="runTest" methodArguments="--workload=transactional" />
 	<arbitraryJava class="net.openj9.test.IdleMicroBenchmark" method="runTest" methodArguments="--workload=image" />
-	<arbitraryJava class="net.openj9.test.IdleMicroBenchmark" method="runTest" methodArguments="--workload=spike" />
+<!--	<arbitraryJava class="net.openj9.test.IdleMicroBenchmark" method="runTest" methodArguments="--workload=spike" /> --!>
 </inventory>

openj9.test.load/src/test.load/net/openj9/stf/IdleLoadTest.java

@@ -1,4 +1,4 @@
-/*******************************************************************************
+/******************************************************************************
 * Copyright (c) 2017 IBM Corp.
 *
 * This program and the accompanying materials are made available under the
@@ -25,23 +25,44 @@
 import net.adoptopenjdk.stf.extensions.core.StfCoreExtension;
 import net.adoptopenjdk.stf.extensions.core.StfCoreExtension.Echo;
 import net.adoptopenjdk.stf.plugin.interfaces.StfPluginInterface;
+import net.adoptopenjdk.stf.environment.StfTestArguments;
 import net.adoptopenjdk.stf.processes.ExpectedOutcome;
 import net.adoptopenjdk.stf.processes.definitions.JavaProcessDefinition;
 import net.adoptopenjdk.stf.processes.definitions.LoadTestProcessDefinition;
 import net.adoptopenjdk.stf.processes.definitions.SystemProcessDefinition;
 import net.adoptopenjdk.stf.runner.modes.HelpTextGenerator;
 
 /**
- * This is a test plugin for Java.util related tests, it runs a workload of 
- * test cases from the test.util project.
+ * This is a test plugin to run the Idle Microbenchmark as a load test which runs 
+ *  workloads from the test.idle project.
  */
 public class IdleLoadTest implements StfPluginInterface {
+	String jvmOptions;
+	String testType;
 	public void help(HelpTextGenerator help) throws StfException {
-		help.outputSection("IdleLoadTest runs a workload of Java.util related tests.");
+		help.outputSection("IdleLoadTest runs a CPU and memory intensive workloads to test working of the Idle tuning feature.");
 		help.outputText("");
 	}
 
 	public void pluginInit(StfCoreExtension test) throws StfException {
+		StfTestArguments testArgs = test.env().getTestProperties("variation=[MinIdleWaitTime]");
+		testType = testArgs.get("variation");
+		if(testType.equals("MinIdleWaitTime")) {
+			jvmOptions = "-XX:IdleTuningMinIdleWaitTime=180 -Xmx1024m -Xjit:verbose={compilePerformance},vlog=jitlog";
+		}
+		else {
+			String platform = test.env().getPlatform();
+			if ( !platform.equals("linux_x86-64") && !platform.equals("linux_x86-32") ) {
+				throw new StfException("This test is only applible on Linux_x86 platforms.");
+			}
+			if ( testType.equals("GcOnIdle") ) {
+				jvmOptions = "-XX:+IdleTuningGcOnIdle -Xtune:virtualized -XX:IdleTuningMinIdleWaitTime=120 -Xmx1024m -verbose:gc -Xverbosegclog:gc.verbose -Xjit:verbose={compilePerformance},vlog=jitlog";
+			}
+			if ( testType.equals("CompactOnIdle") ) {
+				jvmOptions = "-XX:+IdleTuningCompactOnIdle -Xtune:virtualized -XX:IdleTuningMinIdleWaitTime=120 -Xmx1024m -verbose:gc -Xverbosegclog:gc.verbose -Xjit:verbose={compilePerformance},vlog=jitlog";
+			}
+		}
+		
 	}
 
 	public void setUp(StfCoreExtension test) throws StfException {
@@ -52,11 +73,16 @@ public void execute(StfCoreExtension test) throws StfException {
 		test.env().verifyUsingIBMJava();
 
 		String inventoryFile = "/openj9.test.load/config/inventories/idle/idleTest.xml";
-		int threadCount = 3;
+		int threadCount = 2;
 		int numOfIterations = 12;
-		int cpuCount = Runtime.getRuntime().availableProcessors();
+
+		// Numbr of iterations reduced to 6 for MinIdleWait time as 12 iterations cause an OOM with -Xmx=1024m
+		if ( testType.equals("MinIdleWaitTime") ) {
+			numOfIterations = 6;
+		}
 
 		LoadTestProcessDefinition loadTestInvocation = test.createLoadTestSpecification()
+				.addJvmOption(jvmOptions)
 				.addPrereqJarToClasspath(JavaProcessDefinition.JarId.JUNIT)
 				.addPrereqJarToClasspath(JavaProcessDefinition.JarId.HAMCREST)
 				.addProjectToClasspath("openj9.test.idle")
@@ -69,12 +95,18 @@ public void execute(StfCoreExtension test) throws StfException {
 		// Copying the images folder to the results dir
 		test.doCpDir("Copy pics to results dir", test.env().findTestDirectory("openj9.test.idle").childDirectory("src/test.idle/net/openj9/test/images"), test.env().getResultsDir().childDirectory("images"));
 
+		
 		test.doRunForegroundProcess("Run idle load test", "ILT", Echo.ECHO_ON,
 				ExpectedOutcome.cleanRun().within("1h30m"), 
 				loadTestInvocation);
 
 		if ( test.getJavaArgs(test.env().primaryJvm()).contains("-verbose:gc") || test.env().getResultsDir().childFile("gc.verbose").asJavaFile().exists() ) {
-			test.doCountFileMatches("Looking for string release free pages in gc logs", test.env().getResultsDir().childFile("gc.verbose"), numOfIterations, "type=\"release free pages\"");
+			if ( testType.equals("GcOnIdle") ) {
+				test.doCountFileMatches("Looking for string release free pages in gc logs", test.env().getResultsDir().childFile("gc.verbose"), numOfIterations, "type=\"release free pages\"");
+			}
+			if ( testType.equals("CompactOnIdle") ) {
+				test.doCountFileMatches("Looking for string sys-start reason=vm idle in gc logs", test.env().getResultsDir().childFile("gc.verbose"), 1, "sys-start reason=\"vm idle\"");
+			}
 		}
 	}