Skip to content

Commit 71611c3

Browse files
amandelpieamandelpie
authored
Added an approach to differ tests produced by Fuzzer and SymbolicExecutor #581 (#583)
* Handled the bad case with the empty paths * Added the ability to differ tests produced by fuzzing and symbolic engine * Fixed a bug
1 parent a2883ee commit 71611c3

File tree

5 files changed

+85
-33
lines changed

5 files changed

+85
-33
lines changed

‎utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,8 @@ sealed class UtResult
139139
* - static fields changed during execution;
140140
* - required instrumentation details (such as randoms, time, static methods).
141141
* - coverage information (instructions) if this execution was obtained from the concrete execution.
142+
* - the engine type that created this execution.
143+
* - comments, method names and display names created by utbot-summary module.
142144
*/
143145
data class UtExecution(
144146
val stateBefore: EnvironmentModels,
@@ -148,6 +150,7 @@ data class UtExecution(
148150
val path: MutableList<Step>,
149151
val fullPath: List<Step>,
150152
val coverage: Coverage? = null,
153+
val createdBy: UtExecutionCreator? = null,
151154
var summary: List<DocStatement>? = null,
152155
var testMethodName: String? = null,
153156
var displayName: String? = null,
@@ -1172,21 +1175,28 @@ private fun StringBuilder.appendOptional(name: String, value: Map<*, *>) {
11721175
}
11731176

11741177
/**
1175-
* Entity that represents cluster information that should appear in the comment
1178+
* Enum that represents different type of engines that produce tests.
1179+
*/
1180+
enum class UtExecutionCreator {
1181+
FUZZER, SYMBOLIC_ENGINE
1182+
}
1183+
1184+
/**
1185+
* Entity that represents cluster information that should appear in the comment.
11761186
*/
11771187
data class UtClusterInfo(
11781188
val header: String? = null,
11791189
val content: String? = null
11801190
)
11811191

11821192
/**
1183-
* Entity that represents cluster of executions
1193+
* Entity that represents cluster of executions.
11841194
*/
11851195
data class UtExecutionCluster(val clusterInfo: UtClusterInfo, val executions: List<UtExecution>)
11861196

11871197

11881198
/**
1189-
* Entity that represents various types of statements in comments
1199+
* Entity that represents various types of statements in comments.
11901200
*/
11911201
sealed class DocStatement
11921202

‎utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ import org.utbot.framework.plugin.api.UtAssembleModel
6868
import org.utbot.framework.plugin.api.UtConcreteExecutionFailure
6969
import org.utbot.framework.plugin.api.UtError
7070
import org.utbot.framework.plugin.api.UtExecution
71+
import org.utbot.framework.plugin.api.UtExecutionCreator
7172
import org.utbot.framework.plugin.api.UtInstrumentation
7273
import org.utbot.framework.plugin.api.UtMethod
7374
import org.utbot.framework.plugin.api.UtNullModel
@@ -304,7 +305,8 @@ class UtBotSymbolicEngine(
304305
instrumentation,
305306
mutableListOf(),
306307
listOf(),
307-
concreteExecutionResult.coverage
308+
concreteExecutionResult.coverage,
309+
UtExecutionCreator.SYMBOLIC_ENGINE
308310
)
309311
emit(concreteUtExecution)
310312

@@ -487,6 +489,7 @@ class UtBotSymbolicEngine(
487489
path = mutableListOf(),
488490
fullPath = emptyList(),
489491
coverage = concreteExecutionResult.coverage,
492+
createdBy = UtExecutionCreator.FUZZER,
490493
testMethodName = testMethodName?.testName,
491494
displayName = testMethodName?.takeIf { hasMethodUnderTestParametersToFuzz }?.displayName
492495
)
@@ -511,7 +514,8 @@ class UtBotSymbolicEngine(
511514
result = UtConcreteExecutionFailure(e),
512515
instrumentation = emptyList(),
513516
path = mutableListOf(),
514-
fullPath = listOf()
517+
fullPath = listOf(),
518+
createdBy = UtExecutionCreator.SYMBOLIC_ENGINE,
515519
)
516520

517521
emit(failedConcreteExecution)
@@ -546,12 +550,13 @@ class UtBotSymbolicEngine(
546550
require(stateBefore.parameters.size == stateAfter.parameters.size)
547551

548552
val symbolicUtExecution = UtExecution(
549-
stateBefore,
550-
stateAfter,
551-
symbolicExecutionResult,
552-
instrumentation,
553-
entryMethodPath(state),
554-
state.fullPath()
553+
stateBefore = stateBefore,
554+
stateAfter = stateAfter,
555+
result = symbolicExecutionResult,
556+
instrumentation = instrumentation,
557+
path = entryMethodPath(state),
558+
fullPath = state.fullPath(),
559+
createdBy = UtExecutionCreator.SYMBOLIC_ENGINE,
555560
)
556561

557562
globalGraph.traversed(state)

‎utbot-framework/src/main/kotlin/org/utbot/external/api/UtBotJavaApi.kt

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import org.utbot.framework.plugin.api.MockFramework
1818
import org.utbot.framework.plugin.api.MockStrategyApi
1919
import org.utbot.framework.plugin.api.TestCaseGenerator
2020
import org.utbot.framework.plugin.api.UtExecution
21+
import org.utbot.framework.plugin.api.UtExecutionCreator
2122
import org.utbot.framework.plugin.api.UtMethod
2223
import org.utbot.framework.plugin.api.UtPrimitiveModel
2324
import org.utbot.framework.plugin.api.UtMethodTestSet
@@ -240,12 +241,12 @@ object UtBotJavaApi {
240241
}
241242

242243
val utExecution = UtExecution(
243-
testInfo.initialState,
244-
testInfo.initialState, // it seems ok for concrete execution
245-
utExecutionResult,
246-
emptyList(),
247-
mutableListOf(),
248-
listOf()
244+
stateBefore = testInfo.initialState,
245+
stateAfter = testInfo.initialState, // it seems ok for concrete execution
246+
result = utExecutionResult,
247+
instrumentation = emptyList(),
248+
path = mutableListOf(),
249+
fullPath = listOf()
249250
)
250251

251252
val utMethod = UtMethod(methodCallable, containingClass.kotlin)

‎utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt

Lines changed: 51 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import org.utbot.framework.UtSettings
55
import org.utbot.framework.plugin.api.UtClusterInfo
66
import org.utbot.framework.plugin.api.UtExecution
77
import org.utbot.framework.plugin.api.UtExecutionCluster
8+
import org.utbot.framework.plugin.api.UtExecutionCreator
89
import org.utbot.framework.plugin.api.UtMethodTestSet
910
import org.utbot.instrumentation.instrumentation.instrumenter.Instrumenter
1011
import org.utbot.summary.SummarySentenceConstants.NEW_LINE
@@ -75,29 +76,50 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
7576
val updatedExecutions = mutableListOf<UtExecution>()
7677
val clustersToReturn = mutableListOf<UtExecutionCluster>()
7778

78-
// TODO: Now it excludes tests generated by Fuzzer, handle it properly, related to the https://github.com/UnitTestBot/UTBotJava/issues/428
79-
val executionsProducedByFuzzer = getExecutionsWithEmptyPath(testSet)
79+
// handles tests produced by fuzzing
80+
val executionsProducedByFuzzer = testSet.executions.filter { it.createdBy == UtExecutionCreator.FUZZER }
8081

8182
if (executionsProducedByFuzzer.isNotEmpty()) {
8283
executionsProducedByFuzzer.forEach {
8384
logger.info {
84-
"The path for test ${it.testMethodName} " +
85+
"Test is created by Fuzzing. The path for test ${it.testMethodName} " +
8586
"for method ${testSet.method.clazz.qualifiedName} is empty and summaries could not be generated."
8687
}
8788
}
8889

8990
clustersToReturn.add(
9091
UtExecutionCluster(
91-
UtClusterInfo(),
92+
UtClusterInfo(), // TODO: add something https://github.com/UnitTestBot/UTBotJava/issues/430
9293
executionsProducedByFuzzer
9394
)
9495
)
9596
}
9697

98+
// handles tests produced by symbolic engine, but with empty paths
99+
val executionsWithEmptyPaths = getExecutionsCreatedBySymbolicEngineWithEmptyPath(testSet)
100+
101+
if (executionsWithEmptyPaths.isNotEmpty()) {
102+
executionsWithEmptyPaths.forEach {
103+
logger.info {
104+
"Test is created by Symbolic Engine. The path for test ${it.testMethodName} " +
105+
"for method ${testSet.method.clazz.qualifiedName} is empty and summaries could not be generated."
106+
}
107+
}
108+
109+
clustersToReturn.add(
110+
UtExecutionCluster(
111+
UtClusterInfo(), // TODO: https://github.com/UnitTestBot/UTBotJava/issues/430
112+
executionsWithEmptyPaths
113+
)
114+
)
115+
}
116+
117+
val testSetForAnalysis = prepareTestSetForByteCodeAnalysis(testSet)
118+
97119
// analyze
98120
if (jimpleBody != null && sootToAST != null) {
99121
val methodUnderTest = jimpleBody.method
100-
val clusteredTags = tagGenerator.testSetToTags(testSet)
122+
val clusteredTags = tagGenerator.testSetToTags(testSetForAnalysis)
101123
jimpleBodyAnalysis.traceStructuralAnalysis(jimpleBody, clusteredTags, methodUnderTest, invokeDescriptions)
102124
val numberOfSuccessfulClusters = clusteredTags.filter { it.isSuccessful }.size
103125
for (clusterTraceTags in clusteredTags) {
@@ -108,21 +130,22 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
108130
&& clusterTraceTags.traceTags.size > 1 // add if there is more than 1 execution
109131
) {
110132
SimpleClusterCommentBuilder(clusterTraceTags.commonStepsTraceTag, sootToAST)
111-
.buildString(methodUnderTest)
112-
.takeIf { it.isNotBlank() }
113-
?.let {
114-
buildString {
115-
append("${NEW_LINE}Common steps:")
116-
append("$NEW_LINE$it")
117-
}
133+
.buildString(methodUnderTest)
134+
.takeIf { it.isNotBlank() }
135+
?.let {
136+
buildString {
137+
append("${NEW_LINE}Common steps:")
138+
append("$NEW_LINE$it")
118139
}
140+
}
119141
} else {
120142
null
121143
}
122144

123145
for (traceTags in clusterTraceTags.traceTags) {
124146
if (GENERATE_COMMENTS) {
125-
traceTags.execution.summary = SimpleCommentBuilder(traceTags, sootToAST).buildDocStmts(methodUnderTest)
147+
traceTags.execution.summary =
148+
SimpleCommentBuilder(traceTags, sootToAST).buildDocStmts(methodUnderTest)
126149
}
127150

128151
if (GENERATE_DISPLAY_NAMES || GENERATE_NAMES) {
@@ -164,8 +187,21 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
164187
return listOf(UtExecutionCluster(UtClusterInfo(), testSet.executions))
165188
}
166189

167-
private fun getExecutionsWithEmptyPath(testSet: UtMethodTestSet) =
168-
testSet.executions.filter { it.path.isEmpty() }
190+
private fun prepareTestSetForByteCodeAnalysis(testSet: UtMethodTestSet): UtMethodTestSet {
191+
val executions =
192+
testSet.executions.filterNot { it.createdBy == UtExecutionCreator.FUZZER || (it.createdBy == UtExecutionCreator.SYMBOLIC_ENGINE && it.path.isEmpty()) }
193+
194+
return UtMethodTestSet(
195+
method = testSet.method,
196+
executions = executions,
197+
jimpleBody = testSet.jimpleBody,
198+
errors = testSet.errors,
199+
clustersInfo = testSet.clustersInfo
200+
)
201+
}
202+
203+
private fun getExecutionsCreatedBySymbolicEngineWithEmptyPath(testSet: UtMethodTestSet) =
204+
testSet.executions.filter { it.createdBy == UtExecutionCreator.SYMBOLIC_ENGINE && it.path.isEmpty() }
169205

170206
/*
171207
* asts of invokes also included

‎utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ private fun generateExecutionTags(executions: List<UtExecution>, splitSteps: Spl
9595
* @return clustered executions
9696
*/
9797
private fun toClusterExecutions(testSet: UtMethodTestSet): List<ExecutionCluster> {
98-
val methodExecutions = testSet.executions.filter { it.path.isNotEmpty() } // TODO: Now it excludes tests generated by Fuzzer, handle it properly, related to the https://github.com/UnitTestBot/UTBotJava/issues/428
98+
val methodExecutions = testSet.executions
9999
val clusters = mutableListOf<ExecutionCluster>()
100100
val commentPostfix = "for method ${testSet.method.displayName}"
101101

0 commit comments

Comments
 (0)