Non-deterministic detection

utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt

@@ -185,7 +185,12 @@ class UtSymbolicExecution(
         append(")")
     }
 
-    fun copy(stateAfter: EnvironmentModels, result: UtExecutionResult, coverage: Coverage): UtResult {
+    fun copy(
+        stateAfter: EnvironmentModels,
+        result: UtExecutionResult,
+        coverage: Coverage,
+        instrumentation: List<UtInstrumentation> = this.instrumentation,
+    ): UtResult {
         return UtSymbolicExecution(
             stateBefore,
             stateAfter,
@@ -841,6 +846,7 @@ open class ClassId @JvmOverloads constructor(
      */
     open val allMethods: Sequence<MethodId>
         get() = generateSequence(jClass) { it.superclass }
+            .flatMap { it.interfaces.toMutableList() + it }
             .mapNotNull { it.declaredMethods }
             .flatMap { it.toList() }
             .map { it.executableId }

utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt

@@ -262,7 +262,7 @@ class UtBotSymbolicEngine(
                                 stateBefore,
                                 concreteExecutionResult.stateAfter,
                                 concreteExecutionResult.result,
-                                instrumentation,
+                                concreteExecutionResult.newInstrumentation ?: instrumentation,
                                 mutableListOf(),
                                 listOf(),
                                 concreteExecutionResult.coverage
@@ -425,7 +425,8 @@ class UtBotSymbolicEngine(
                     result = concreteExecutionResult.result,
                     coverage = concreteExecutionResult.coverage,
                     fuzzingValues = values,
-                    fuzzedMethodDescription = descr.description
+                    fuzzedMethodDescription = descr.description,
+                    instrumentation = concreteExecutionResult.newInstrumentation ?: emptyList()
                 )
             )
 
@@ -552,7 +553,8 @@ class UtBotSymbolicEngine(
                 val concolicUtExecution = symbolicUtExecution.copy(
                     stateAfter = concreteExecutionResult.stateAfter,
                     result = concreteExecutionResult.result,
-                    coverage = concreteExecutionResult.coverage
+                    coverage = concreteExecutionResult.coverage,
+                    instrumentation = concreteExecutionResult.newInstrumentation ?: instrumentation
                 )
 
                 emit(concolicUtExecution)

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/tree/CgMethodConstructor.kt

@@ -149,6 +149,7 @@ import org.utbot.framework.plugin.api.util.stringClassId
 import org.utbot.framework.plugin.api.util.voidClassId
 import org.utbot.framework.plugin.api.util.wrapIfPrimitive
 import org.utbot.framework.util.isUnit
+import org.utbot.fuzzer.UtFuzzedExecution
 import org.utbot.summary.SummarySentenceConstants.TAB
 import java.lang.reflect.InvocationTargetException
 import java.lang.reflect.ParameterizedType
@@ -184,29 +185,30 @@ open class CgMethodConstructor(val context: CgContext) : CgContextOwner by conte
     private var containsStreamConsumingFailureForParametrizedTests: Boolean = false
 
     protected fun setupInstrumentation() {
-        if (currentExecution is UtSymbolicExecution) {
-            val execution = currentExecution as UtSymbolicExecution
-            val instrumentation = execution.instrumentation
-            if (instrumentation.isEmpty()) return
-
-            if (generateWarningsForStaticMocking && forceStaticMocking == ForceStaticMocking.DO_NOT_FORCE) {
-                // warn user about possible flaky tests
-                multilineComment(forceStaticMocking.warningMessage)
-                return
-            }
+        val instrumentation = when (val execution = currentExecution) {
+            is UtSymbolicExecution -> execution.instrumentation
+            is UtFuzzedExecution -> execution.instrumentation
+            else -> return
+        }
+        if (instrumentation.isEmpty()) return
 
-            instrumentation
-                .filterIsInstance<UtNewInstanceInstrumentation>()
-                .forEach { mockFrameworkManager.mockNewInstance(it) }
-            instrumentation
-                .filterIsInstance<UtStaticMethodInstrumentation>()
-                .groupBy { it.methodId.classId }
-                .forEach { (classId, methodMocks) -> mockFrameworkManager.mockStaticMethodsOfClass(classId, methodMocks) }
-
-            if (generateWarningsForStaticMocking && forceStaticMocking == ForceStaticMocking.FORCE) {
-                // warn user about forced using static mocks
-                multilineComment(forceStaticMocking.warningMessage)
-            }
+        if (generateWarningsForStaticMocking && forceStaticMocking == ForceStaticMocking.DO_NOT_FORCE) {
+            // warn user about possible flaky tests
+            multilineComment(forceStaticMocking.warningMessage)
+            return
+        }
+
+        instrumentation
+            .filterIsInstance<UtNewInstanceInstrumentation>()
+            .forEach { mockFrameworkManager.mockNewInstance(it) }
+        instrumentation
+            .filterIsInstance<UtStaticMethodInstrumentation>()
+            .groupBy { it.methodId.classId }
+            .forEach { (classId, methodMocks) -> mockFrameworkManager.mockStaticMethodsOfClass(classId, methodMocks) }
+
+        if (generateWarningsForStaticMocking && forceStaticMocking == ForceStaticMocking.FORCE) {
+            // warn user about forced using static mocks
+            multilineComment(forceStaticMocking.warningMessage)
         }
     }
 

utbot-framework/src/main/kotlin/org/utbot/framework/util/UtConcreteExecutionResultUtils.kt

@@ -24,7 +24,8 @@ private fun UtConcreteExecutionResult.updateWithAssembleModels(
     return UtConcreteExecutionResult(
         resolvedStateAfter,
         resolvedResult,
-        coverage
+        coverage,
+        newInstrumentation
     )
 }
 

utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/UtExecutionInstrumentation.kt

@@ -1,22 +1,24 @@
 package org.utbot.instrumentation.instrumentation.execution
 
-import java.security.ProtectionDomain
-import java.util.IdentityHashMap
-import kotlin.reflect.jvm.javaMethod
 import org.utbot.framework.UtSettings
 import org.utbot.framework.plugin.api.*
-import org.utbot.instrumentation.instrumentation.execution.constructors.ConstructOnlyUserClassesOrCachedObjectsStrategy
-import org.utbot.instrumentation.instrumentation.execution.constructors.UtModelConstructor
-import org.utbot.instrumentation.instrumentation.execution.mock.InstrumentationContext
-import org.utbot.instrumentation.instrumentation.execution.phases.PhasesController
-import org.utbot.instrumentation.instrumentation.execution.phases.start
 import org.utbot.framework.plugin.api.util.singleExecutableId
 import org.utbot.instrumentation.instrumentation.ArgumentList
 import org.utbot.instrumentation.instrumentation.Instrumentation
 import org.utbot.instrumentation.instrumentation.InvokeInstrumentation
 import org.utbot.instrumentation.instrumentation.et.TraceHandler
+import org.utbot.instrumentation.instrumentation.execution.constructors.ConstructOnlyUserClassesOrCachedObjectsStrategy
+import org.utbot.instrumentation.instrumentation.execution.constructors.UtModelConstructor
+import org.utbot.instrumentation.instrumentation.execution.mock.InstrumentationContext
+import org.utbot.instrumentation.instrumentation.execution.ndd.NonDeterministicClassVisitor
+import org.utbot.instrumentation.instrumentation.execution.ndd.NonDeterministicDetector
+import org.utbot.instrumentation.instrumentation.execution.phases.PhasesController
+import org.utbot.instrumentation.instrumentation.execution.phases.start
 import org.utbot.instrumentation.instrumentation.instrumenter.Instrumenter
 import org.utbot.instrumentation.instrumentation.mock.MockClassVisitor
+import java.security.ProtectionDomain
+import java.util.*
+import kotlin.reflect.jvm.javaMethod
 
 /**
  * Consists of the data needed to execute the method concretely. Also includes method arguments stored in models.
@@ -35,7 +37,8 @@ data class UtConcreteExecutionData(
 class UtConcreteExecutionResult(
     val stateAfter: EnvironmentModels,
     val result: UtExecutionResult,
-    val coverage: Coverage
+    val coverage: Coverage,
+    val newInstrumentation: List<UtInstrumentation>? = null,
 ) {
     override fun toString(): String = buildString {
         appendLine("UtConcreteExecutionResult(")
@@ -51,6 +54,7 @@ object UtExecutionInstrumentation : Instrumentation<UtConcreteExecutionResult> {
     private val instrumentationContext = InstrumentationContext()
 
     private val traceHandler = TraceHandler()
+    private val ndDetector = NonDeterministicDetector()
     private val pathsToUserClasses = mutableSetOf<String>()
 
     override fun init(pathsToUserClasses: Set<String>) {
@@ -101,6 +105,7 @@ object UtExecutionInstrumentation : Instrumentation<UtConcreteExecutionResult> {
                 postprocessingPhase.setStaticFields(preparationPhase.start {
                     val result = setStaticFields(statics)
                     resetTrace()
+                    resetND()
                     result
                 })
 
@@ -110,12 +115,12 @@ object UtExecutionInstrumentation : Instrumentation<UtConcreteExecutionResult> {
                 }
 
                 // statistics collection
-                val coverage = executePhaseInTimeout(statisticsCollectionPhase) {
-                    getCoverage(clazz)
+                val (coverage, ndResults) = executePhaseInTimeout(statisticsCollectionPhase) {
+                    getCoverage(clazz) to getNonDeterministicResults()
                 }
 
                 // model construction
-                val (executionResult, stateAfter) = executePhaseInTimeout(modelConstructionPhase) {
+                val (executionResult, stateAfter, newInstrumentation) = executePhaseInTimeout(modelConstructionPhase) {
                     configureConstructor {
                         this.cache = cache
                         strategy = ConstructOnlyUserClassesOrCachedObjectsStrategy(
@@ -124,6 +129,10 @@ object UtExecutionInstrumentation : Instrumentation<UtConcreteExecutionResult> {
                         )
                     }
 
+                    val ndStatics = constructStaticInstrumentation(ndResults.statics)
+                    val ndNews = constructNewInstrumentation(ndResults.news, ndResults.calls)
+                    val newInstrumentation = mergeInstrumentations(instrumentations, ndStatics, ndNews)
+
                     val executionResult = convertToExecutionResult(concreteResult, returnClassId)
 
                     val stateAfterParametersWithThis = constructParameters(params)
@@ -135,13 +144,14 @@ object UtExecutionInstrumentation : Instrumentation<UtConcreteExecutionResult> {
                     }
                     val stateAfter = EnvironmentModels(stateAfterThis, stateAfterParameters, stateAfterStatics)
 
-                    executionResult to stateAfter
+                    Triple(executionResult, stateAfter, newInstrumentation)
                 }
 
                 UtConcreteExecutionResult(
                     stateAfter,
                     executionResult,
-                    coverage
+                    coverage,
+                    newInstrumentation
                 )
             } finally {
                 postprocessingPhase.start {
@@ -175,6 +185,10 @@ object UtExecutionInstrumentation : Instrumentation<UtConcreteExecutionResult> {
         traceHandler.registerClass(className)
         instrumenter.visitInstructions(traceHandler.computeInstructionVisitor(className))
 
+        instrumenter.visitClass { writer ->
+            NonDeterministicClassVisitor(writer, ndDetector)
+        }
+
         val mockClassVisitor = instrumenter.visitClass { writer ->
             MockClassVisitor(
                 writer,

utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/constructors/UtModelConstructor.kt

@@ -1,40 +1,13 @@
 package org.utbot.instrumentation.instrumentation.execution.constructors
 
-import java.lang.reflect.Modifier
-import java.util.IdentityHashMap
-import java.util.stream.BaseStream
 import org.utbot.common.asPathToFile
 import org.utbot.common.withAccessibility
-import org.utbot.framework.plugin.api.ClassId
-import org.utbot.framework.plugin.api.FieldId
-import org.utbot.framework.plugin.api.UtArrayModel
-import org.utbot.framework.plugin.api.UtAssembleModel
-import org.utbot.framework.plugin.api.UtClassRefModel
-import org.utbot.framework.plugin.api.UtCompositeModel
-import org.utbot.framework.plugin.api.UtEnumConstantModel
-import org.utbot.framework.plugin.api.UtLambdaModel
-import org.utbot.framework.plugin.api.UtModel
-import org.utbot.framework.plugin.api.UtNullModel
-import org.utbot.framework.plugin.api.UtPrimitiveModel
-import org.utbot.framework.plugin.api.UtReferenceModel
-import org.utbot.framework.plugin.api.UtVoidModel
-import org.utbot.framework.plugin.api.isMockModel
-import org.utbot.framework.plugin.api.util.booleanClassId
-import org.utbot.framework.plugin.api.util.byteClassId
-import org.utbot.framework.plugin.api.util.charClassId
-import org.utbot.framework.plugin.api.util.doubleClassId
-import org.utbot.framework.plugin.api.util.fieldId
-import org.utbot.framework.plugin.api.util.floatClassId
-import org.utbot.framework.plugin.api.util.id
-import org.utbot.framework.plugin.api.util.intClassId
-import org.utbot.framework.plugin.api.util.isInaccessibleViaReflection
-import org.utbot.framework.plugin.api.util.isPrimitive
-import org.utbot.framework.plugin.api.util.jClass
-import org.utbot.framework.plugin.api.util.longClassId
-import org.utbot.framework.plugin.api.util.objectClassId
-import org.utbot.framework.plugin.api.util.shortClassId
-import org.utbot.framework.plugin.api.util.utContext
+import org.utbot.framework.plugin.api.*
+import org.utbot.framework.plugin.api.util.*
 import org.utbot.framework.plugin.api.visible.UtStreamConsumingException
+import java.lang.reflect.Modifier
+import java.util.*
+import java.util.stream.BaseStream
 
 /**
  * Represents common interface for model constructors.
@@ -120,6 +93,7 @@ class UtModelConstructor(
             is Float,
             is Double,
             is Boolean -> if (classId.isPrimitive) UtPrimitiveModel(value) else constructFromAny(value)
+
             is ByteArray -> constructFromByteArray(value)
             is ShortArray -> constructFromShortArray(value)
             is CharArray -> constructFromCharArray(value)
@@ -136,6 +110,20 @@ class UtModelConstructor(
         }
     }
 
+    fun constructMock(instance: Any, classId: ClassId, mocks: Map<MethodId, List<Any?>>): UtModel =
+        constructedObjects.getOrElse(instance) {
+            val utModel = UtCompositeModel(
+                handleId(instance),
+                classId,
+                isMock = true,
+                mocks = mocks.mapValuesTo(mutableMapOf()) { (method, values) ->
+                    values.map { construct(it, method.returnType) }
+                }
+            )
+            constructedObjects[instance] = utModel
+            utModel
+        }
+
     // Q: Is there a way to get rid of duplicated code?
 
     private fun constructFromDoubleArray(array: DoubleArray): UtModel =

utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicBytecodeInserter.kt

@@ -0,0 +1,85 @@
+package org.utbot.instrumentation.instrumentation.execution.ndd
+
+import org.objectweb.asm.MethodVisitor
+import org.objectweb.asm.Opcodes
+import org.objectweb.asm.Type
+
+class NonDeterministicBytecodeInserter {
+    private val internalName = Type.getInternalName(NonDeterministicResultStorage::class.java)
+
+    private fun String.getUnifiedParamsTypes(): List<String> {
+        val list = mutableListOf<String>()
+        var readObject = false
+        for (c in this) {
+            if (c == '(') {
+                continue
+            }
+            if (c == ')') {
+                break
+            }
+            if (readObject) {
+                if (c == ';') {
+                    readObject = false
+                    list.add("Ljava/lang/Object;")
+                }
+            } else if (c == 'L') {
+                readObject = true
+            } else {
+                list.add(c.toString())
+            }
+        }
+
+        return list
+    }
+
+    private fun String.unifyTypeDescriptor(): String =
+        if (startsWith('L')) {
+            "Ljava/lang/Object;"
+        } else {
+            this
+        }
+
+    private fun String.getReturnType(): String =
+        substringAfter(')')
+
+    private fun getStoreDescriptor(descriptor: String): String = buildString {
+        append('(')
+        append(descriptor.getReturnType().unifyTypeDescriptor())
+        append("Ljava/lang/String;)V")
+    }
+
+    private fun MethodVisitor.invoke(name: String, descriptor: String) {
+        visitMethodInsn(Opcodes.INVOKESTATIC, internalName, name, descriptor, false)
+    }
+
+    fun insertAfterNDMethod(mv: MethodVisitor, owner: String, name: String, descriptor: String, isStatic: Boolean) {
+        mv.visitInsn(Opcodes.DUP)
+        mv.visitLdcInsn(NonDeterministicResultStorage.makeSignature(owner, name, descriptor))
+        mv.invoke(if (isStatic) "storeStatic" else "storeCall", getStoreDescriptor(descriptor))
+    }
+
+    fun insertBeforeNDMethod(mv: MethodVisitor, descriptor: String, isStatic: Boolean) {
+        if (isStatic) {
+            return
+        }
+
+        val params = descriptor.getUnifiedParamsTypes()
+
+        params.asReversed().forEach {
+            mv.invoke("putParameter${it[0]}", "($it)V")
+        }
+
+        mv.visitInsn(Opcodes.DUP)
+        mv.invoke("saveInstance", "(Ljava/lang/Object;)V")
+
+        params.forEach {
+            mv.invoke("peakParameter${it[0]}", "()$it")
+        }
+    }
+
+    fun insertAfterNDInstanceConstructor(mv: MethodVisitor, callSite: String) {
+        mv.visitInsn(Opcodes.DUP)
+        mv.visitLdcInsn(callSite)
+        mv.invoke("registerInstance", "(Ljava/lang/Object;Ljava/lang/String;)V")
+    }
+}