From 844b4d295b2f9293e4ae7029ebd1a0db13844230 Mon Sep 17 00:00:00 2001 From: Rustam Sadykov Date: Fri, 5 May 2023 16:57:33 +0200 Subject: [PATCH 1/5] non-deterministic detection version 1 --- .../org/utbot/framework/plugin/api/Api.kt | 7 +- .../org/utbot/engine/UtBotSymbolicEngine.kt | 8 +- .../codegen/tree/CgMethodConstructor.kt | 46 +++++----- .../util/UtConcreteExecutionResultUtils.kt | 3 +- .../execution/UtExecutionInstrumentation.kt | 42 ++++++---- .../constructors/UtModelConstructor.kt | 52 +++++------- .../ndd/NonDeterministicBytecodeInserter.kt | 63 ++++++++++++++ .../ndd/NonDeterministicClassVisitor.kt | 84 +++++++++++++++++++ .../execution/ndd/NonDeterministicDetector.kt | 57 +++++++++++++ .../ndd/NonDeterministicResultStorage.kt | 71 ++++++++++++++++ .../phases/ModelConstructionPhase.kt | 50 +++++++++-- .../execution/phases/PreparationPhase.kt | 5 ++ .../phases/StatisticsCollectionPhase.kt | 40 +++++++++ .../org/utbot/fuzzer/UtFuzzedExecution.kt | 10 +-- 14 files changed, 452 insertions(+), 86 deletions(-) create mode 100644 utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicBytecodeInserter.kt create mode 100644 utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicClassVisitor.kt create mode 100644 utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicDetector.kt create mode 100644 utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicResultStorage.kt diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt index eb837766d6..079d716bfe 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt @@ -185,7 +185,12 @@ class UtSymbolicExecution( append(")") } - fun copy(stateAfter: EnvironmentModels, result: UtExecutionResult, coverage: Coverage): UtResult { + fun copy( + stateAfter: EnvironmentModels, + result: UtExecutionResult, + coverage: Coverage, + instrumentation: List = this.instrumentation, + ): UtResult { return UtSymbolicExecution( stateBefore, stateAfter, diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index 87c54616e7..7033434b5a 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -262,7 +262,7 @@ class UtBotSymbolicEngine( stateBefore, concreteExecutionResult.stateAfter, concreteExecutionResult.result, - instrumentation, + concreteExecutionResult.newInstrumentation ?: instrumentation, mutableListOf(), listOf(), concreteExecutionResult.coverage @@ -425,7 +425,8 @@ class UtBotSymbolicEngine( result = concreteExecutionResult.result, coverage = concreteExecutionResult.coverage, fuzzingValues = values, - fuzzedMethodDescription = descr.description + fuzzedMethodDescription = descr.description, + instrumentation = concreteExecutionResult.newInstrumentation ?: emptyList() ) ) @@ -552,7 +553,8 @@ class UtBotSymbolicEngine( val concolicUtExecution = symbolicUtExecution.copy( stateAfter = concreteExecutionResult.stateAfter, result = concreteExecutionResult.result, - coverage = concreteExecutionResult.coverage + coverage = concreteExecutionResult.coverage, + instrumentation = concreteExecutionResult.newInstrumentation ?: instrumentation ) emit(concolicUtExecution) diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/tree/CgMethodConstructor.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/tree/CgMethodConstructor.kt index 0d16cd710e..4d177cc573 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/tree/CgMethodConstructor.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/tree/CgMethodConstructor.kt @@ -149,6 +149,7 @@ import org.utbot.framework.plugin.api.util.stringClassId import org.utbot.framework.plugin.api.util.voidClassId import org.utbot.framework.plugin.api.util.wrapIfPrimitive import org.utbot.framework.util.isUnit +import org.utbot.fuzzer.UtFuzzedExecution import org.utbot.summary.SummarySentenceConstants.TAB import java.lang.reflect.InvocationTargetException import java.lang.reflect.ParameterizedType @@ -184,29 +185,30 @@ open class CgMethodConstructor(val context: CgContext) : CgContextOwner by conte private var containsStreamConsumingFailureForParametrizedTests: Boolean = false protected fun setupInstrumentation() { - if (currentExecution is UtSymbolicExecution) { - val execution = currentExecution as UtSymbolicExecution - val instrumentation = execution.instrumentation - if (instrumentation.isEmpty()) return - - if (generateWarningsForStaticMocking && forceStaticMocking == ForceStaticMocking.DO_NOT_FORCE) { - // warn user about possible flaky tests - multilineComment(forceStaticMocking.warningMessage) - return - } + val instrumentation = when (val execution = currentExecution) { + is UtSymbolicExecution -> execution.instrumentation + is UtFuzzedExecution -> execution.instrumentation + else -> return + } + if (instrumentation.isEmpty()) return - instrumentation - .filterIsInstance() - .forEach { mockFrameworkManager.mockNewInstance(it) } - instrumentation - .filterIsInstance() - .groupBy { it.methodId.classId } - .forEach { (classId, methodMocks) -> mockFrameworkManager.mockStaticMethodsOfClass(classId, methodMocks) } - - if (generateWarningsForStaticMocking && forceStaticMocking == ForceStaticMocking.FORCE) { - // warn user about forced using static mocks - multilineComment(forceStaticMocking.warningMessage) - } + if (generateWarningsForStaticMocking && forceStaticMocking == ForceStaticMocking.DO_NOT_FORCE) { + // warn user about possible flaky tests + multilineComment(forceStaticMocking.warningMessage) + return + } + + instrumentation + .filterIsInstance() + .forEach { mockFrameworkManager.mockNewInstance(it) } + instrumentation + .filterIsInstance() + .groupBy { it.methodId.classId } + .forEach { (classId, methodMocks) -> mockFrameworkManager.mockStaticMethodsOfClass(classId, methodMocks) } + + if (generateWarningsForStaticMocking && forceStaticMocking == ForceStaticMocking.FORCE) { + // warn user about forced using static mocks + multilineComment(forceStaticMocking.warningMessage) } } diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/util/UtConcreteExecutionResultUtils.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/util/UtConcreteExecutionResultUtils.kt index 2ed5c100eb..9aef5f1f99 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/framework/util/UtConcreteExecutionResultUtils.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/framework/util/UtConcreteExecutionResultUtils.kt @@ -24,7 +24,8 @@ private fun UtConcreteExecutionResult.updateWithAssembleModels( return UtConcreteExecutionResult( resolvedStateAfter, resolvedResult, - coverage + coverage, + newInstrumentation ) } diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/UtExecutionInstrumentation.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/UtExecutionInstrumentation.kt index 823786cf87..b1bc40e565 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/UtExecutionInstrumentation.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/UtExecutionInstrumentation.kt @@ -1,22 +1,24 @@ package org.utbot.instrumentation.instrumentation.execution -import java.security.ProtectionDomain -import java.util.IdentityHashMap -import kotlin.reflect.jvm.javaMethod import org.utbot.framework.UtSettings import org.utbot.framework.plugin.api.* -import org.utbot.instrumentation.instrumentation.execution.constructors.ConstructOnlyUserClassesOrCachedObjectsStrategy -import org.utbot.instrumentation.instrumentation.execution.constructors.UtModelConstructor -import org.utbot.instrumentation.instrumentation.execution.mock.InstrumentationContext -import org.utbot.instrumentation.instrumentation.execution.phases.PhasesController -import org.utbot.instrumentation.instrumentation.execution.phases.start import org.utbot.framework.plugin.api.util.singleExecutableId import org.utbot.instrumentation.instrumentation.ArgumentList import org.utbot.instrumentation.instrumentation.Instrumentation import org.utbot.instrumentation.instrumentation.InvokeInstrumentation import org.utbot.instrumentation.instrumentation.et.TraceHandler +import org.utbot.instrumentation.instrumentation.execution.constructors.ConstructOnlyUserClassesOrCachedObjectsStrategy +import org.utbot.instrumentation.instrumentation.execution.constructors.UtModelConstructor +import org.utbot.instrumentation.instrumentation.execution.mock.InstrumentationContext +import org.utbot.instrumentation.instrumentation.execution.ndd.NonDeterministicClassVisitor +import org.utbot.instrumentation.instrumentation.execution.ndd.NonDeterministicDetector +import org.utbot.instrumentation.instrumentation.execution.phases.PhasesController +import org.utbot.instrumentation.instrumentation.execution.phases.start import org.utbot.instrumentation.instrumentation.instrumenter.Instrumenter import org.utbot.instrumentation.instrumentation.mock.MockClassVisitor +import java.security.ProtectionDomain +import java.util.* +import kotlin.reflect.jvm.javaMethod /** * Consists of the data needed to execute the method concretely. Also includes method arguments stored in models. @@ -35,7 +37,8 @@ data class UtConcreteExecutionData( class UtConcreteExecutionResult( val stateAfter: EnvironmentModels, val result: UtExecutionResult, - val coverage: Coverage + val coverage: Coverage, + val newInstrumentation: List? = null, ) { override fun toString(): String = buildString { appendLine("UtConcreteExecutionResult(") @@ -51,6 +54,7 @@ object UtExecutionInstrumentation : Instrumentation { private val instrumentationContext = InstrumentationContext() private val traceHandler = TraceHandler() + private val ndDetector = NonDeterministicDetector() private val pathsToUserClasses = mutableSetOf() override fun init(pathsToUserClasses: Set) { @@ -101,6 +105,7 @@ object UtExecutionInstrumentation : Instrumentation { postprocessingPhase.setStaticFields(preparationPhase.start { val result = setStaticFields(statics) resetTrace() + resetND() result }) @@ -110,12 +115,12 @@ object UtExecutionInstrumentation : Instrumentation { } // statistics collection - val coverage = executePhaseInTimeout(statisticsCollectionPhase) { - getCoverage(clazz) + val (coverage, ndResults) = executePhaseInTimeout(statisticsCollectionPhase) { + getCoverage(clazz) to getNonDeterministicResults() } // model construction - val (executionResult, stateAfter) = executePhaseInTimeout(modelConstructionPhase) { + val (executionResult, stateAfter, newInstrumentation) = executePhaseInTimeout(modelConstructionPhase) { configureConstructor { this.cache = cache strategy = ConstructOnlyUserClassesOrCachedObjectsStrategy( @@ -124,6 +129,10 @@ object UtExecutionInstrumentation : Instrumentation { ) } + val ndStatics = constructStaticInstrumentation(ndResults.statics) + val ndNews = constructNewInstrumentation(ndResults.news, ndResults.calls) + val newInstrumentation = mergeInstrumentations(instrumentations, ndStatics, ndNews) + val executionResult = convertToExecutionResult(concreteResult, returnClassId) val stateAfterParametersWithThis = constructParameters(params) @@ -135,13 +144,14 @@ object UtExecutionInstrumentation : Instrumentation { } val stateAfter = EnvironmentModels(stateAfterThis, stateAfterParameters, stateAfterStatics) - executionResult to stateAfter + Triple(executionResult, stateAfter, newInstrumentation) } UtConcreteExecutionResult( stateAfter, executionResult, - coverage + coverage, + newInstrumentation ) } finally { postprocessingPhase.start { @@ -175,6 +185,10 @@ object UtExecutionInstrumentation : Instrumentation { traceHandler.registerClass(className) instrumenter.visitInstructions(traceHandler.computeInstructionVisitor(className)) + instrumenter.visitClass { writer -> + NonDeterministicClassVisitor(writer, ndDetector) + } + val mockClassVisitor = instrumenter.visitClass { writer -> MockClassVisitor( writer, diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/constructors/UtModelConstructor.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/constructors/UtModelConstructor.kt index f020382c85..f49a523f5f 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/constructors/UtModelConstructor.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/constructors/UtModelConstructor.kt @@ -1,40 +1,13 @@ package org.utbot.instrumentation.instrumentation.execution.constructors -import java.lang.reflect.Modifier -import java.util.IdentityHashMap -import java.util.stream.BaseStream import org.utbot.common.asPathToFile import org.utbot.common.withAccessibility -import org.utbot.framework.plugin.api.ClassId -import org.utbot.framework.plugin.api.FieldId -import org.utbot.framework.plugin.api.UtArrayModel -import org.utbot.framework.plugin.api.UtAssembleModel -import org.utbot.framework.plugin.api.UtClassRefModel -import org.utbot.framework.plugin.api.UtCompositeModel -import org.utbot.framework.plugin.api.UtEnumConstantModel -import org.utbot.framework.plugin.api.UtLambdaModel -import org.utbot.framework.plugin.api.UtModel -import org.utbot.framework.plugin.api.UtNullModel -import org.utbot.framework.plugin.api.UtPrimitiveModel -import org.utbot.framework.plugin.api.UtReferenceModel -import org.utbot.framework.plugin.api.UtVoidModel -import org.utbot.framework.plugin.api.isMockModel -import org.utbot.framework.plugin.api.util.booleanClassId -import org.utbot.framework.plugin.api.util.byteClassId -import org.utbot.framework.plugin.api.util.charClassId -import org.utbot.framework.plugin.api.util.doubleClassId -import org.utbot.framework.plugin.api.util.fieldId -import org.utbot.framework.plugin.api.util.floatClassId -import org.utbot.framework.plugin.api.util.id -import org.utbot.framework.plugin.api.util.intClassId -import org.utbot.framework.plugin.api.util.isInaccessibleViaReflection -import org.utbot.framework.plugin.api.util.isPrimitive -import org.utbot.framework.plugin.api.util.jClass -import org.utbot.framework.plugin.api.util.longClassId -import org.utbot.framework.plugin.api.util.objectClassId -import org.utbot.framework.plugin.api.util.shortClassId -import org.utbot.framework.plugin.api.util.utContext +import org.utbot.framework.plugin.api.* +import org.utbot.framework.plugin.api.util.* import org.utbot.framework.plugin.api.visible.UtStreamConsumingException +import java.lang.reflect.Modifier +import java.util.* +import java.util.stream.BaseStream /** * Represents common interface for model constructors. @@ -120,6 +93,7 @@ class UtModelConstructor( is Float, is Double, is Boolean -> if (classId.isPrimitive) UtPrimitiveModel(value) else constructFromAny(value) + is ByteArray -> constructFromByteArray(value) is ShortArray -> constructFromShortArray(value) is CharArray -> constructFromCharArray(value) @@ -136,6 +110,20 @@ class UtModelConstructor( } } + fun constructMock(instance: Any, classId: ClassId, mocks: Map>): UtModel = + constructedObjects.getOrElse(instance) { + val utModel = UtCompositeModel( + handleId(instance), + classId, + isMock = true, + mocks = mocks.mapValuesTo(mutableMapOf()) { (method, values) -> + values.map { construct(it, method.returnType) } + } + ) + constructedObjects[instance] = utModel + utModel + } + // Q: Is there a way to get rid of duplicated code? private fun constructFromDoubleArray(array: DoubleArray): UtModel = diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicBytecodeInserter.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicBytecodeInserter.kt new file mode 100644 index 0000000000..48af745e3e --- /dev/null +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicBytecodeInserter.kt @@ -0,0 +1,63 @@ +package org.utbot.instrumentation.instrumentation.execution.ndd + +import org.objectweb.asm.MethodVisitor +import org.objectweb.asm.Opcodes +import org.objectweb.asm.Type +import org.utbot.framework.plugin.api.ClassId +import org.utbot.framework.plugin.api.MethodId +import org.utbot.framework.plugin.api.util.* + +class NonDeterministicBytecodeInserter { + private val internalName = Type.getInternalName(NonDeterministicResultStorage::class.java) + + private fun ClassId.descriptor(): String = when (this) { + booleanClassId -> "Z" + byteClassId -> "B" + charClassId -> "C" + shortClassId -> "S" + intClassId -> "I" + longClassId -> "J" + floatClassId -> "F" + doubleClassId -> "D" + else -> "Ljava/lang/Object;" + } + + private fun MethodId.toStoreDescriptor(): String = buildString { + append('(') + append(returnType.descriptor()) + append("Ljava/lang/String;)V") + } + + private fun MethodVisitor.invoke(name: String, descriptor: String) { + visitMethodInsn(Opcodes.INVOKESTATIC, internalName, name, descriptor, false) + } + + fun insertAfterNDMethod(mv: MethodVisitor, methodId: MethodId) { + mv.visitInsn(Opcodes.DUP) + mv.visitLdcInsn(NonDeterministicResultStorage.methodToSignature(methodId)) + mv.invoke(if (methodId.isStatic) "storeStatic" else "storeCall", methodId.toStoreDescriptor()) + } + + fun insertBeforeNDMethod(mv: MethodVisitor, methodId: MethodId) { + if (methodId.isStatic) { + return + } + + methodId.parameters.asReversed().forEach { + mv.invoke("putParameter", "(${it.descriptor()})V") + } + + mv.visitInsn(Opcodes.DUP) + mv.invoke("saveInstance", "(Ljava/lang/Object;)V") + + methodId.parameters.forEach { + mv.invoke("peakParameter", "()${it.descriptor()}") + } + } + + fun insertAfterNDInstanceConstructor(mv: MethodVisitor, callSite: String) { + mv.visitInsn(Opcodes.DUP) + mv.visitLdcInsn(callSite) + mv.invoke("registerInstance", "(Ljava/lang/Object;Ljava/lang/String;)V") + } +} \ No newline at end of file diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicClassVisitor.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicClassVisitor.kt new file mode 100644 index 0000000000..c01f78d36d --- /dev/null +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicClassVisitor.kt @@ -0,0 +1,84 @@ +package org.utbot.instrumentation.instrumentation.execution.ndd + +import org.objectweb.asm.ClassVisitor +import org.objectweb.asm.MethodVisitor +import org.objectweb.asm.commons.Method +import org.utbot.framework.plugin.api.MethodId +import org.utbot.framework.plugin.api.util.executableId +import org.utbot.framework.plugin.api.util.id +import org.utbot.framework.plugin.api.util.utContext +import org.utbot.instrumentation.Settings + +class NonDeterministicClassVisitor( + classVisitor: ClassVisitor, + private val detector: NonDeterministicDetector +) : ClassVisitor(Settings.ASM_API, classVisitor) { + + private var currentClass: String? = null + + private fun getOwnerClass(owner: String): Class<*> = + utContext.classLoader.loadClass(owner.replace('/', '.')) + + private fun getMethodId(owner: String?, name: String?, descriptor: String?): MethodId? { + if (owner == null || name == null || descriptor == null) { + return null + } + val clazz = getOwnerClass(owner) + val method = clazz.methods.find { + it.name == name && Method.getMethod(it).descriptor == descriptor + } + return method?.executableId + } + + override fun visit( + version: Int, + access: Int, + name: String?, + signature: String?, + superName: String?, + interfaces: Array? + ) { + currentClass = name + super.visit(version, access, name, signature, superName, interfaces) + } + + override fun visitMethod( + access: Int, + name: String?, + descriptor: String?, + signature: String?, + exceptions: Array? + ): MethodVisitor { + val mv = cv.visitMethod(access, name, descriptor, signature, exceptions) + return object : MethodVisitor(Settings.ASM_API, mv) { + override fun visitMethodInsn( + opcodeAndSource: Int, + owner: String?, + name: String?, + descriptor: String?, + isInterface: Boolean + ) { + if (name == "") { + mv.visitMethodInsn(opcodeAndSource, owner, name, descriptor, isInterface) + owner?.let { + if (detector.isNonDeterministic(getOwnerClass(it).id)) { + detector.inserter.insertAfterNDInstanceConstructor(mv, currentClass!!) + } + } + return + } + + getMethodId(owner, name, descriptor)?.let { method -> + if (detector.isNonDeterministic(method)) { + detector.inserter.insertBeforeNDMethod(mv, method) + mv.visitMethodInsn(opcodeAndSource, owner, name, descriptor, isInterface) + detector.inserter.insertAfterNDMethod(mv, method) + return + } + } + + mv.visitMethodInsn(opcodeAndSource, owner, name, descriptor, isInterface) + } + } + } +} \ No newline at end of file diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicDetector.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicDetector.kt new file mode 100644 index 0000000000..0773902c4a --- /dev/null +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicDetector.kt @@ -0,0 +1,57 @@ +package org.utbot.instrumentation.instrumentation.execution.ndd + +import org.utbot.framework.plugin.api.ClassId +import org.utbot.framework.plugin.api.MethodId +import org.utbot.framework.plugin.api.util.executableId +import org.utbot.framework.plugin.api.util.id +import java.lang.reflect.Method +import java.security.SecureRandom +import kotlin.random.Random + +class NonDeterministicDetector { + + private fun MutableList.addJavaRandomMethods() { + addAll(java.util.Random::class.java.methods + .filter { it.name.startsWith("next") && it.returnType?.isPrimitive == true } + ) + } + + private fun MutableList.addKotlinRandomMethods() { + val badMethods = hashSetOf("nextBits", "nextBytes", "nextUBytes") + addAll(Random::class.java.methods + .filter { it.name.startsWith("next") && it.returnType?.isPrimitive == true } + .filterNot { badMethods.contains(it.name) } + ) + } + + private val nonDeterministicMethods: HashSet = buildList { + addJavaRandomMethods() + addKotlinRandomMethods() + }.map { it.executableId }.toHashSet() + + private val nonDeterministicClasses: HashSet = buildList { + add(java.util.Random::class.java) + add(Random::class.java) + }.map { it.id }.toHashSet() + + val inserter = NonDeterministicBytecodeInserter() + + fun isNonDeterministic(method: MethodId): Boolean { + var mt: MethodId? = method + while (mt != null && !nonDeterministicMethods.contains(mt)) { + mt = method.classId.superclass?.let { clazz -> + clazz.id.allMethods.find { it.signature == method.signature } + } + } + return nonDeterministicMethods.contains(mt) + } + + fun isNonDeterministic(clazz: ClassId): Boolean { + var cl: ClassId? = clazz + while (cl != null && !nonDeterministicClasses.contains(cl)) { + cl = cl.superclass?.id + } + return nonDeterministicClasses.contains(cl) + } + +} \ No newline at end of file diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicResultStorage.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicResultStorage.kt new file mode 100644 index 0000000000..67b02bf9fb --- /dev/null +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicResultStorage.kt @@ -0,0 +1,71 @@ +@file:Suppress("UNUSED") + +package org.utbot.instrumentation.instrumentation.execution.ndd + +import org.utbot.framework.plugin.api.MethodId +import org.utbot.framework.plugin.api.util.id +import org.utbot.framework.plugin.api.util.utContext +import java.util.IdentityHashMap + +object NonDeterministicResultStorage { + + data class NDMethodResult(val signature: String, val result: Any?) + data class NDInstanceInfo(val instanceNumber: Int, val callSite: String) + + private var currentInstance: Any? = null + private val parameters: MutableList = mutableListOf() + + val staticStorage: MutableList = mutableListOf() + val callStorage: IdentityHashMap> = IdentityHashMap() + val ndInstances: IdentityHashMap = IdentityHashMap() + private var nextInstanceNumber = 1 + + fun clear() { + staticStorage.clear() + callStorage.clear() + ndInstances.clear() + nextInstanceNumber = 1 + } + + fun methodToSignature(methodId: MethodId): String { + return "${methodId.classId.name} ${methodId.signature}" + } + + fun signatureToMethod(signature: String): MethodId? { + val sign = signature.split(' ') + val clazz = utContext.classLoader.loadClass( + sign[0].replace('/', '.') + ).id + return clazz.allMethods.find { it.signature == sign[1] } + } + + @JvmStatic + fun registerInstance(instance: Any, callSite: String) { + ndInstances[instance] = NDInstanceInfo(nextInstanceNumber++, callSite) + } + + @JvmStatic + fun saveInstance(instance: Any) { + currentInstance = instance + } + + @JvmStatic + fun putParameter(value: Int) { + parameters.add(value) + } + + @JvmStatic + fun peakParameter(): Int { + return parameters.removeLast() as Int + } + + @JvmStatic + fun storeStatic(result: Int, signature: String) { + staticStorage.add(NDMethodResult(signature, result)) + } + + @JvmStatic + fun storeCall(result: Int, signature: String) { + callStorage.getOrPut(currentInstance) { mutableListOf() }.add(NDMethodResult(signature, result)) + } +} diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/ModelConstructionPhase.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/ModelConstructionPhase.kt index 29d380104d..0132e906b8 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/ModelConstructionPhase.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/ModelConstructionPhase.kt @@ -1,17 +1,17 @@ package org.utbot.instrumentation.instrumentation.execution.phases -import java.security.AccessControlException -import java.util.IdentityHashMap import org.utbot.common.withAccessibility import org.utbot.framework.plugin.api.* -import org.utbot.instrumentation.instrumentation.execution.constructors.UtCompositeModelStrategy -import org.utbot.instrumentation.instrumentation.execution.constructors.UtModelConstructor import org.utbot.framework.plugin.api.util.id import org.utbot.framework.plugin.api.util.jField import org.utbot.framework.plugin.api.visible.UtStreamConsumingException import org.utbot.instrumentation.instrumentation.et.ExplicitThrowInstruction import org.utbot.instrumentation.instrumentation.et.TraceHandler import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult +import org.utbot.instrumentation.instrumentation.execution.constructors.UtCompositeModelStrategy +import org.utbot.instrumentation.instrumentation.execution.constructors.UtModelConstructor +import java.security.AccessControlException +import java.util.* /** * This phase of model construction from concrete values. @@ -22,8 +22,12 @@ class ModelConstructionPhase( override fun wrapError(e: Throwable): ExecutionPhaseException { val message = this.javaClass.simpleName - return when(e) { - is TimeoutException -> ExecutionPhaseStop(message, UtConcreteExecutionResult(MissingState, UtTimeoutException(e), Coverage())) + return when (e) { + is TimeoutException -> ExecutionPhaseStop( + message, + UtConcreteExecutionResult(MissingState, UtTimeoutException(e), Coverage()) + ) + else -> ExecutionPhaseError(message, e) } } @@ -42,6 +46,40 @@ class ModelConstructionPhase( } } + fun mergeInstrumentations( + oldInstrumentations: List, + statics: List, + news: List + ): List = mutableListOf().apply { + val st = statics.associateBy { it.methodId } + val nw = news.associateBy { it.classId } + + addAll(oldInstrumentations.filterNot { + when (it) { + is UtStaticMethodInstrumentation -> st.contains(it.methodId) + is UtNewInstanceInstrumentation -> nw.contains(it.classId) + } + }) + addAll(statics) + addAll(news) + } + + fun constructStaticInstrumentation(statics: Map>): List = + statics.map { (method, values) -> + UtStaticMethodInstrumentation(method, values.map { constructor.construct(it, method.returnType) }) + } + + fun constructNewInstrumentation( + news: Map, Set>>, + calls: IdentityHashMap>>, + ): List = news.map { (classId, info) -> + val models = info.first.map { instance -> + constructor.constructMock(instance, classId, calls[instance] ?: emptyMap()) + } + + UtNewInstanceInstrumentation(classId, models, info.second) + } + fun constructParameters(params: List>): List = params.map { constructor.construct(it.value, it.clazz.id) diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/PreparationPhase.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/PreparationPhase.kt index 1576947380..a733192fd5 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/PreparationPhase.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/PreparationPhase.kt @@ -5,6 +5,7 @@ import org.utbot.framework.plugin.api.FieldId import org.utbot.framework.plugin.api.UtConcreteValue import org.utbot.framework.plugin.api.util.jField import org.utbot.instrumentation.instrumentation.et.TraceHandler +import org.utbot.instrumentation.instrumentation.execution.ndd.NonDeterministicResultStorage /** @@ -34,4 +35,8 @@ class PreparationPhase( traceHandler.resetTrace() } + fun resetND() { + NonDeterministicResultStorage.clear() + } + } \ No newline at end of file diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/StatisticsCollectionPhase.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/StatisticsCollectionPhase.kt index d3264cbf70..d9b3d1acac 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/StatisticsCollectionPhase.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/StatisticsCollectionPhase.kt @@ -2,9 +2,14 @@ package org.utbot.instrumentation.instrumentation.execution.phases import org.objectweb.asm.Type import org.utbot.framework.plugin.api.* +import org.utbot.framework.plugin.api.util.id +import org.utbot.framework.plugin.api.util.utContext import org.utbot.instrumentation.instrumentation.et.EtInstruction import org.utbot.instrumentation.instrumentation.et.TraceHandler import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult +import org.utbot.instrumentation.instrumentation.execution.ndd.NonDeterministicResultStorage +import org.utbot.instrumentation.process.HandlerClassesLoader +import java.util.* /** * This phase is about collection statistics such as coverage. @@ -21,6 +26,41 @@ class StatisticsCollectionPhase( } } + data class NDResults( + val statics: Map>, + val news: Map, Set>>, + val calls: IdentityHashMap>> + ) + + fun getNonDeterministicResults() : NDResults { + val storage = NonDeterministicResultStorage + + val statics = storage.staticStorage + .groupBy { storage.signatureToMethod(it.signature)!! } + .mapValues { (_, values) -> values.map { it.result } } + + val news = try { + storage.ndInstances.entries + .groupBy { it.key.javaClass.id } + .mapValues { (_, entries) -> + val values = entries.sortedBy { it.value.instanceNumber }.map { it.key } + val callSites = entries.map { + utContext.classLoader.loadClass(it.value.callSite.replace('/', '.')).id + }.toSet() + values to callSites + } + } catch (e: Throwable) { + throw e + } + + val calls = storage.callStorage.mapValuesTo(IdentityHashMap()) { (_, methodResults) -> methodResults + .groupBy { storage.signatureToMethod(it.signature)!! } + .mapValues { (_, values) -> values.map { it.result } } + } + + return NDResults(statics, news, calls) + } + fun getCoverage(clazz: Class<*>): Coverage { return traceHandler .computeInstructionList() diff --git a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzer/UtFuzzedExecution.kt b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzer/UtFuzzedExecution.kt index 211ec0fcfb..a158aa7259 100644 --- a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzer/UtFuzzedExecution.kt +++ b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzer/UtFuzzedExecution.kt @@ -1,11 +1,6 @@ package org.utbot.fuzzer -import org.utbot.framework.plugin.api.Coverage -import org.utbot.framework.plugin.api.DocStatement -import org.utbot.framework.plugin.api.EnvironmentModels -import org.utbot.framework.plugin.api.FieldId -import org.utbot.framework.plugin.api.UtExecution -import org.utbot.framework.plugin.api.UtExecutionResult +import org.utbot.framework.plugin.api.* /** * Fuzzed execution. @@ -26,7 +21,8 @@ class UtFuzzedExecution( testMethodName: String? = null, displayName: String? = null, val fuzzingValues: List? = null, - val fuzzedMethodDescription: FuzzedMethodDescription? = null + val fuzzedMethodDescription: FuzzedMethodDescription? = null, + val instrumentation: List = emptyList(), ) : UtExecution(stateBefore, stateAfter, result, coverage, summary, testMethodName, displayName) { /** * By design the 'before' and 'after' states contain info about the same fields. From f464eebcaad3b0828b19b2e71eba492777a6daac Mon Sep 17 00:00:00 2001 From: Rustam Sadykov Date: Sun, 14 May 2023 14:24:23 +0200 Subject: [PATCH 2/5] add support longs and objects --- .../ndd/NonDeterministicBytecodeInserter.kt | 6 ++- .../ndd/NonDeterministicResultStorage.kt | 46 ++++++++++++++++++- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicBytecodeInserter.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicBytecodeInserter.kt index 48af745e3e..22d2a177f9 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicBytecodeInserter.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicBytecodeInserter.kt @@ -44,14 +44,16 @@ class NonDeterministicBytecodeInserter { } methodId.parameters.asReversed().forEach { - mv.invoke("putParameter", "(${it.descriptor()})V") + val desc = it.descriptor() + mv.invoke("putParameter${desc[0]}", "($desc)V") } mv.visitInsn(Opcodes.DUP) mv.invoke("saveInstance", "(Ljava/lang/Object;)V") methodId.parameters.forEach { - mv.invoke("peakParameter", "()${it.descriptor()}") + val desc = it.descriptor() + mv.invoke("peakParameter${desc[0]}", "()$desc") } } diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicResultStorage.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicResultStorage.kt index 67b02bf9fb..24fe0353d5 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicResultStorage.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicResultStorage.kt @@ -7,6 +7,8 @@ import org.utbot.framework.plugin.api.util.id import org.utbot.framework.plugin.api.util.utContext import java.util.IdentityHashMap +// TODO: refactor using code generation +// TODO: support all primitives object NonDeterministicResultStorage { data class NDMethodResult(val signature: String, val result: Any?) @@ -50,22 +52,62 @@ object NonDeterministicResultStorage { } @JvmStatic - fun putParameter(value: Int) { + fun putParameterI(value: Int) { parameters.add(value) } @JvmStatic - fun peakParameter(): Int { + fun peakParameterI(): Int { return parameters.removeLast() as Int } + @JvmStatic + fun putParameterJ(value: Long) { + parameters.add(value) + } + + @JvmStatic + fun peakParameterJ(): Long { + return parameters.removeLast() as Long + } + + @JvmStatic + fun putParameterL(value: Any?) { + parameters.add(value) + } + + @JvmStatic + fun peakParameterL(): Any? { + return parameters.removeLast() + } + @JvmStatic fun storeStatic(result: Int, signature: String) { staticStorage.add(NDMethodResult(signature, result)) } + @JvmStatic + fun storeStatic(result: Long, signature: String) { + staticStorage.add(NDMethodResult(signature, result)) + } + + @JvmStatic + fun storeStatic(result: Any?, signature: String) { + staticStorage.add(NDMethodResult(signature, result)) + } + @JvmStatic fun storeCall(result: Int, signature: String) { callStorage.getOrPut(currentInstance) { mutableListOf() }.add(NDMethodResult(signature, result)) } + + @JvmStatic + fun storeCall(result: Long, signature: String) { + callStorage.getOrPut(currentInstance) { mutableListOf() }.add(NDMethodResult(signature, result)) + } + + @JvmStatic + fun storeCall(result: Any?, signature: String) { + callStorage.getOrPut(currentInstance) { mutableListOf() }.add(NDMethodResult(signature, result)) + } } From 4a2188af6e8dafc97b98784fece2a1518240dd18 Mon Sep 17 00:00:00 2001 From: Rustam Sadykov Date: Wed, 31 May 2023 10:29:38 +0200 Subject: [PATCH 3/5] add rest primitives --- .../execution/ndd/NonDeterministicDetector.kt | 32 +---- .../ndd/NonDeterministicResultStorage.kt | 132 +++++++++++++++++- .../phases/ModelConstructionPhase.kt | 8 +- .../phases/StatisticsCollectionPhase.kt | 43 +++--- 4 files changed, 161 insertions(+), 54 deletions(-) diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicDetector.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicDetector.kt index 0773902c4a..eb9a754623 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicDetector.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicDetector.kt @@ -4,30 +4,12 @@ import org.utbot.framework.plugin.api.ClassId import org.utbot.framework.plugin.api.MethodId import org.utbot.framework.plugin.api.util.executableId import org.utbot.framework.plugin.api.util.id +import org.utbot.framework.plugin.api.util.isStatic import java.lang.reflect.Method -import java.security.SecureRandom import kotlin.random.Random class NonDeterministicDetector { - - private fun MutableList.addJavaRandomMethods() { - addAll(java.util.Random::class.java.methods - .filter { it.name.startsWith("next") && it.returnType?.isPrimitive == true } - ) - } - - private fun MutableList.addKotlinRandomMethods() { - val badMethods = hashSetOf("nextBits", "nextBytes", "nextUBytes") - addAll(Random::class.java.methods - .filter { it.name.startsWith("next") && it.returnType?.isPrimitive == true } - .filterNot { badMethods.contains(it.name) } - ) - } - - private val nonDeterministicMethods: HashSet = buildList { - addJavaRandomMethods() - addKotlinRandomMethods() - }.map { it.executableId }.toHashSet() + private val nonDeterministicStaticMethods: HashSet = HashSet() private val nonDeterministicClasses: HashSet = buildList { add(java.util.Random::class.java) @@ -37,13 +19,11 @@ class NonDeterministicDetector { val inserter = NonDeterministicBytecodeInserter() fun isNonDeterministic(method: MethodId): Boolean { - var mt: MethodId? = method - while (mt != null && !nonDeterministicMethods.contains(mt)) { - mt = method.classId.superclass?.let { clazz -> - clazz.id.allMethods.find { it.signature == method.signature } - } + return if (method.isStatic) { + nonDeterministicStaticMethods.contains(method) + } else { + isNonDeterministic(method.classId) } - return nonDeterministicMethods.contains(mt) } fun isNonDeterministic(clazz: ClassId): Boolean { diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicResultStorage.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicResultStorage.kt index 24fe0353d5..e3683c3d26 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicResultStorage.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicResultStorage.kt @@ -5,10 +5,9 @@ package org.utbot.instrumentation.instrumentation.execution.ndd import org.utbot.framework.plugin.api.MethodId import org.utbot.framework.plugin.api.util.id import org.utbot.framework.plugin.api.util.utContext -import java.util.IdentityHashMap +import java.util.* + -// TODO: refactor using code generation -// TODO: support all primitives object NonDeterministicResultStorage { data class NDMethodResult(val signature: String, val result: Any?) @@ -51,6 +50,49 @@ object NonDeterministicResultStorage { currentInstance = instance } + // putParameter[type](type) + // peakParameter[type](): type + + @JvmStatic + fun putParameterZ(value: Boolean) { + parameters.add(value) + } + + @JvmStatic + fun peakParameterZ(): Boolean { + return parameters.removeLast() as Boolean + } + + @JvmStatic + fun putParameterB(value: Byte) { + parameters.add(value) + } + + @JvmStatic + fun peakParameterB(): Byte { + return parameters.removeLast() as Byte + } + + @JvmStatic + fun putParameterC(value: Char) { + parameters.add(value) + } + + @JvmStatic + fun peakParameterC(): Char { + return parameters.removeLast() as Char + } + + @JvmStatic + fun putParameterS(value: Short) { + parameters.add(value) + } + + @JvmStatic + fun peakParameterS(): Short { + return parameters.removeLast() as Short + } + @JvmStatic fun putParameterI(value: Int) { parameters.add(value) @@ -71,6 +113,26 @@ object NonDeterministicResultStorage { return parameters.removeLast() as Long } + @JvmStatic + fun putParameterF(value: Float) { + parameters.add(value) + } + + @JvmStatic + fun peakParameterF(): Float { + return parameters.removeLast() as Float + } + + @JvmStatic + fun putParameterD(value: Double) { + parameters.add(value) + } + + @JvmStatic + fun peakParameterD(): Double { + return parameters.removeLast() as Double + } + @JvmStatic fun putParameterL(value: Any?) { parameters.add(value) @@ -81,6 +143,28 @@ object NonDeterministicResultStorage { return parameters.removeLast() } + // storeStatic(type, sign) + + @JvmStatic + fun storeStatic(result: Boolean, signature: String) { + staticStorage.add(NDMethodResult(signature, result)) + } + + @JvmStatic + fun storeStatic(result: Byte, signature: String) { + staticStorage.add(NDMethodResult(signature, result)) + } + + @JvmStatic + fun storeStatic(result: Char, signature: String) { + staticStorage.add(NDMethodResult(signature, result)) + } + + @JvmStatic + fun storeStatic(result: Short, signature: String) { + staticStorage.add(NDMethodResult(signature, result)) + } + @JvmStatic fun storeStatic(result: Int, signature: String) { staticStorage.add(NDMethodResult(signature, result)) @@ -91,11 +175,43 @@ object NonDeterministicResultStorage { staticStorage.add(NDMethodResult(signature, result)) } + @JvmStatic + fun storeStatic(result: Float, signature: String) { + staticStorage.add(NDMethodResult(signature, result)) + } + + @JvmStatic + fun storeStatic(result: Double, signature: String) { + staticStorage.add(NDMethodResult(signature, result)) + } + @JvmStatic fun storeStatic(result: Any?, signature: String) { staticStorage.add(NDMethodResult(signature, result)) } + // storeCall(type, sign) + + @JvmStatic + fun storeCall(result: Boolean, signature: String) { + callStorage.getOrPut(currentInstance) { mutableListOf() }.add(NDMethodResult(signature, result)) + } + + @JvmStatic + fun storeCall(result: Byte, signature: String) { + callStorage.getOrPut(currentInstance) { mutableListOf() }.add(NDMethodResult(signature, result)) + } + + @JvmStatic + fun storeCall(result: Char, signature: String) { + callStorage.getOrPut(currentInstance) { mutableListOf() }.add(NDMethodResult(signature, result)) + } + + @JvmStatic + fun storeCall(result: Short, signature: String) { + callStorage.getOrPut(currentInstance) { mutableListOf() }.add(NDMethodResult(signature, result)) + } + @JvmStatic fun storeCall(result: Int, signature: String) { callStorage.getOrPut(currentInstance) { mutableListOf() }.add(NDMethodResult(signature, result)) @@ -106,6 +222,16 @@ object NonDeterministicResultStorage { callStorage.getOrPut(currentInstance) { mutableListOf() }.add(NDMethodResult(signature, result)) } + @JvmStatic + fun storeCall(result: Float, signature: String) { + callStorage.getOrPut(currentInstance) { mutableListOf() }.add(NDMethodResult(signature, result)) + } + + @JvmStatic + fun storeCall(result: Double, signature: String) { + callStorage.getOrPut(currentInstance) { mutableListOf() }.add(NDMethodResult(signature, result)) + } + @JvmStatic fun storeCall(result: Any?, signature: String) { callStorage.getOrPut(currentInstance) { mutableListOf() }.add(NDMethodResult(signature, result)) diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/ModelConstructionPhase.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/ModelConstructionPhase.kt index 0132e906b8..2c28010251 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/ModelConstructionPhase.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/ModelConstructionPhase.kt @@ -51,13 +51,13 @@ class ModelConstructionPhase( statics: List, news: List ): List = mutableListOf().apply { - val st = statics.associateBy { it.methodId } - val nw = news.associateBy { it.classId } + val method2Static = statics.associateBy { it.methodId } + val class2New = news.associateBy { it.classId } addAll(oldInstrumentations.filterNot { when (it) { - is UtStaticMethodInstrumentation -> st.contains(it.methodId) - is UtNewInstanceInstrumentation -> nw.contains(it.classId) + is UtStaticMethodInstrumentation -> method2Static.contains(it.methodId) + is UtNewInstanceInstrumentation -> class2New.contains(it.classId) } }) addAll(statics) diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/StatisticsCollectionPhase.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/StatisticsCollectionPhase.kt index d9b3d1acac..a5c75a1f18 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/StatisticsCollectionPhase.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/StatisticsCollectionPhase.kt @@ -8,7 +8,6 @@ import org.utbot.instrumentation.instrumentation.et.EtInstruction import org.utbot.instrumentation.instrumentation.et.TraceHandler import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult import org.utbot.instrumentation.instrumentation.execution.ndd.NonDeterministicResultStorage -import org.utbot.instrumentation.process.HandlerClassesLoader import java.util.* /** @@ -20,8 +19,12 @@ class StatisticsCollectionPhase( override fun wrapError(e: Throwable): ExecutionPhaseException { val message = this.javaClass.simpleName - return when(e) { - is TimeoutException -> ExecutionPhaseStop(message, UtConcreteExecutionResult(MissingState, UtTimeoutException(e), Coverage())) + return when (e) { + is TimeoutException -> ExecutionPhaseStop( + message, + UtConcreteExecutionResult(MissingState, UtTimeoutException(e), Coverage()) + ) + else -> ExecutionPhaseError(message, e) } } @@ -32,31 +35,29 @@ class StatisticsCollectionPhase( val calls: IdentityHashMap>> ) - fun getNonDeterministicResults() : NDResults { + fun getNonDeterministicResults(): NDResults { val storage = NonDeterministicResultStorage val statics = storage.staticStorage .groupBy { storage.signatureToMethod(it.signature)!! } .mapValues { (_, values) -> values.map { it.result } } - val news = try { - storage.ndInstances.entries - .groupBy { it.key.javaClass.id } - .mapValues { (_, entries) -> - val values = entries.sortedBy { it.value.instanceNumber }.map { it.key } - val callSites = entries.map { - utContext.classLoader.loadClass(it.value.callSite.replace('/', '.')).id - }.toSet() - values to callSites - } - } catch (e: Throwable) { - throw e - } + val news = storage.ndInstances.entries + .groupBy { it.key.javaClass.id } + .mapValues { (_, entries) -> + val values = entries.sortedBy { it.value.instanceNumber }.map { it.key } + val callSites = entries.map { + utContext.classLoader.loadClass(it.value.callSite.replace('/', '.')).id + }.toSet() + values to callSites + } - val calls = storage.callStorage.mapValuesTo(IdentityHashMap()) { (_, methodResults) -> methodResults - .groupBy { storage.signatureToMethod(it.signature)!! } - .mapValues { (_, values) -> values.map { it.result } } - } + val calls = storage.callStorage + .mapValuesTo(IdentityHashMap()) { (_, methodResults) -> + methodResults + .groupBy { storage.signatureToMethod(it.signature)!! } + .mapValues { (_, values) -> values.map { it.result } } + } return NDResults(statics, news, calls) } From ef6a6ba5948dcadcdb8f1920e5a6d4368833693c Mon Sep 17 00:00:00 2001 From: Rustam Sadykov Date: Wed, 31 May 2023 11:20:40 +0200 Subject: [PATCH 4/5] fix nd checking --- .../execution/ndd/NonDeterministicClassVisitor.kt | 11 ++++++----- .../execution/ndd/NonDeterministicDetector.kt | 6 ++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicClassVisitor.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicClassVisitor.kt index c01f78d36d..0b5ee34fc7 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicClassVisitor.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicClassVisitor.kt @@ -3,6 +3,7 @@ package org.utbot.instrumentation.instrumentation.execution.ndd import org.objectweb.asm.ClassVisitor import org.objectweb.asm.MethodVisitor import org.objectweb.asm.commons.Method +import org.utbot.framework.plugin.api.ClassId import org.utbot.framework.plugin.api.MethodId import org.utbot.framework.plugin.api.util.executableId import org.utbot.framework.plugin.api.util.id @@ -19,15 +20,15 @@ class NonDeterministicClassVisitor( private fun getOwnerClass(owner: String): Class<*> = utContext.classLoader.loadClass(owner.replace('/', '.')) - private fun getMethodId(owner: String?, name: String?, descriptor: String?): MethodId? { + private fun getMethodAndOwnerId(owner: String?, name: String?, descriptor: String?): Pair? { if (owner == null || name == null || descriptor == null) { return null } val clazz = getOwnerClass(owner) val method = clazz.methods.find { it.name == name && Method.getMethod(it).descriptor == descriptor - } - return method?.executableId + } ?: return null + return clazz.id to method.executableId } override fun visit( @@ -68,8 +69,8 @@ class NonDeterministicClassVisitor( return } - getMethodId(owner, name, descriptor)?.let { method -> - if (detector.isNonDeterministic(method)) { + getMethodAndOwnerId(owner, name, descriptor)?.let { (caller, method) -> + if (detector.isNonDeterministic(caller, method)) { detector.inserter.insertBeforeNDMethod(mv, method) mv.visitMethodInsn(opcodeAndSource, owner, name, descriptor, isInterface) detector.inserter.insertAfterNDMethod(mv, method) diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicDetector.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicDetector.kt index eb9a754623..041c32c9f6 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicDetector.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicDetector.kt @@ -2,10 +2,8 @@ package org.utbot.instrumentation.instrumentation.execution.ndd import org.utbot.framework.plugin.api.ClassId import org.utbot.framework.plugin.api.MethodId -import org.utbot.framework.plugin.api.util.executableId import org.utbot.framework.plugin.api.util.id import org.utbot.framework.plugin.api.util.isStatic -import java.lang.reflect.Method import kotlin.random.Random class NonDeterministicDetector { @@ -18,11 +16,11 @@ class NonDeterministicDetector { val inserter = NonDeterministicBytecodeInserter() - fun isNonDeterministic(method: MethodId): Boolean { + fun isNonDeterministic(caller: ClassId, method: MethodId): Boolean { return if (method.isStatic) { nonDeterministicStaticMethods.contains(method) } else { - isNonDeterministic(method.classId) + isNonDeterministic(caller) } } From e741e96b60bfe74ab84bf13569c54679128574b0 Mon Sep 17 00:00:00 2001 From: Rustam Sadykov Date: Wed, 31 May 2023 15:02:24 +0200 Subject: [PATCH 5/5] fix nd without loading class --- .../org/utbot/framework/plugin/api/Api.kt | 1 + .../ndd/NonDeterministicBytecodeInserter.kt | 72 ++++++++++++------- .../ndd/NonDeterministicClassVisitor.kt | 61 ++++++---------- .../execution/ndd/NonDeterministicDetector.kt | 32 +++------ .../ndd/NonDeterministicResultStorage.kt | 4 +- 5 files changed, 80 insertions(+), 90 deletions(-) diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt index 079d716bfe..483ac079b7 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt @@ -846,6 +846,7 @@ open class ClassId @JvmOverloads constructor( */ open val allMethods: Sequence get() = generateSequence(jClass) { it.superclass } + .flatMap { it.interfaces.toMutableList() + it } .mapNotNull { it.declaredMethods } .flatMap { it.toList() } .map { it.executableId } diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicBytecodeInserter.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicBytecodeInserter.kt index 22d2a177f9..39607565a0 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicBytecodeInserter.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicBytecodeInserter.kt @@ -3,28 +3,48 @@ package org.utbot.instrumentation.instrumentation.execution.ndd import org.objectweb.asm.MethodVisitor import org.objectweb.asm.Opcodes import org.objectweb.asm.Type -import org.utbot.framework.plugin.api.ClassId -import org.utbot.framework.plugin.api.MethodId -import org.utbot.framework.plugin.api.util.* class NonDeterministicBytecodeInserter { private val internalName = Type.getInternalName(NonDeterministicResultStorage::class.java) - private fun ClassId.descriptor(): String = when (this) { - booleanClassId -> "Z" - byteClassId -> "B" - charClassId -> "C" - shortClassId -> "S" - intClassId -> "I" - longClassId -> "J" - floatClassId -> "F" - doubleClassId -> "D" - else -> "Ljava/lang/Object;" + private fun String.getUnifiedParamsTypes(): List { + val list = mutableListOf() + var readObject = false + for (c in this) { + if (c == '(') { + continue + } + if (c == ')') { + break + } + if (readObject) { + if (c == ';') { + readObject = false + list.add("Ljava/lang/Object;") + } + } else if (c == 'L') { + readObject = true + } else { + list.add(c.toString()) + } + } + + return list } - private fun MethodId.toStoreDescriptor(): String = buildString { + private fun String.unifyTypeDescriptor(): String = + if (startsWith('L')) { + "Ljava/lang/Object;" + } else { + this + } + + private fun String.getReturnType(): String = + substringAfter(')') + + private fun getStoreDescriptor(descriptor: String): String = buildString { append('(') - append(returnType.descriptor()) + append(descriptor.getReturnType().unifyTypeDescriptor()) append("Ljava/lang/String;)V") } @@ -32,28 +52,28 @@ class NonDeterministicBytecodeInserter { visitMethodInsn(Opcodes.INVOKESTATIC, internalName, name, descriptor, false) } - fun insertAfterNDMethod(mv: MethodVisitor, methodId: MethodId) { + fun insertAfterNDMethod(mv: MethodVisitor, owner: String, name: String, descriptor: String, isStatic: Boolean) { mv.visitInsn(Opcodes.DUP) - mv.visitLdcInsn(NonDeterministicResultStorage.methodToSignature(methodId)) - mv.invoke(if (methodId.isStatic) "storeStatic" else "storeCall", methodId.toStoreDescriptor()) + mv.visitLdcInsn(NonDeterministicResultStorage.makeSignature(owner, name, descriptor)) + mv.invoke(if (isStatic) "storeStatic" else "storeCall", getStoreDescriptor(descriptor)) } - fun insertBeforeNDMethod(mv: MethodVisitor, methodId: MethodId) { - if (methodId.isStatic) { + fun insertBeforeNDMethod(mv: MethodVisitor, descriptor: String, isStatic: Boolean) { + if (isStatic) { return } - methodId.parameters.asReversed().forEach { - val desc = it.descriptor() - mv.invoke("putParameter${desc[0]}", "($desc)V") + val params = descriptor.getUnifiedParamsTypes() + + params.asReversed().forEach { + mv.invoke("putParameter${it[0]}", "($it)V") } mv.visitInsn(Opcodes.DUP) mv.invoke("saveInstance", "(Ljava/lang/Object;)V") - methodId.parameters.forEach { - val desc = it.descriptor() - mv.invoke("peakParameter${desc[0]}", "()$desc") + params.forEach { + mv.invoke("peakParameter${it[0]}", "()$it") } } diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicClassVisitor.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicClassVisitor.kt index 0b5ee34fc7..83ac459594 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicClassVisitor.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicClassVisitor.kt @@ -2,12 +2,7 @@ package org.utbot.instrumentation.instrumentation.execution.ndd import org.objectweb.asm.ClassVisitor import org.objectweb.asm.MethodVisitor -import org.objectweb.asm.commons.Method -import org.utbot.framework.plugin.api.ClassId -import org.utbot.framework.plugin.api.MethodId -import org.utbot.framework.plugin.api.util.executableId -import org.utbot.framework.plugin.api.util.id -import org.utbot.framework.plugin.api.util.utContext +import org.objectweb.asm.Opcodes import org.utbot.instrumentation.Settings class NonDeterministicClassVisitor( @@ -15,26 +10,12 @@ class NonDeterministicClassVisitor( private val detector: NonDeterministicDetector ) : ClassVisitor(Settings.ASM_API, classVisitor) { - private var currentClass: String? = null - - private fun getOwnerClass(owner: String): Class<*> = - utContext.classLoader.loadClass(owner.replace('/', '.')) - - private fun getMethodAndOwnerId(owner: String?, name: String?, descriptor: String?): Pair? { - if (owner == null || name == null || descriptor == null) { - return null - } - val clazz = getOwnerClass(owner) - val method = clazz.methods.find { - it.name == name && Method.getMethod(it).descriptor == descriptor - } ?: return null - return clazz.id to method.executableId - } + private lateinit var currentClass: String override fun visit( version: Int, access: Int, - name: String?, + name: String, signature: String?, superName: String?, interfaces: Array? @@ -45,8 +26,8 @@ class NonDeterministicClassVisitor( override fun visitMethod( access: Int, - name: String?, - descriptor: String?, + name: String, + descriptor: String, signature: String?, exceptions: Array? ): MethodVisitor { @@ -54,31 +35,33 @@ class NonDeterministicClassVisitor( return object : MethodVisitor(Settings.ASM_API, mv) { override fun visitMethodInsn( opcodeAndSource: Int, - owner: String?, - name: String?, - descriptor: String?, + owner: String, + name: String, + descriptor: String, isInterface: Boolean ) { if (name == "") { mv.visitMethodInsn(opcodeAndSource, owner, name, descriptor, isInterface) - owner?.let { - if (detector.isNonDeterministic(getOwnerClass(it).id)) { - detector.inserter.insertAfterNDInstanceConstructor(mv, currentClass!!) - } + if (detector.isNonDeterministicClass(owner)) { + detector.inserter.insertAfterNDInstanceConstructor(mv, currentClass) } return } - getMethodAndOwnerId(owner, name, descriptor)?.let { (caller, method) -> - if (detector.isNonDeterministic(caller, method)) { - detector.inserter.insertBeforeNDMethod(mv, method) - mv.visitMethodInsn(opcodeAndSource, owner, name, descriptor, isInterface) - detector.inserter.insertAfterNDMethod(mv, method) - return - } + val (isND, isStatic) = if (opcodeAndSource == Opcodes.INVOKESTATIC) { + detector.isNonDeterministicStaticFunction(owner, name, descriptor) to true + } else { + detector.isNonDeterministicClass(owner) to false + } + + if (isND) { + detector.inserter.insertBeforeNDMethod(mv, descriptor, isStatic) + mv.visitMethodInsn(opcodeAndSource, owner, name, descriptor, isInterface) + detector.inserter.insertAfterNDMethod(mv, owner, name, descriptor, isStatic) + } else { + mv.visitMethodInsn(opcodeAndSource, owner, name, descriptor, isInterface) } - mv.visitMethodInsn(opcodeAndSource, owner, name, descriptor, isInterface) } } } diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicDetector.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicDetector.kt index 041c32c9f6..d6ef35c2be 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicDetector.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicDetector.kt @@ -1,35 +1,21 @@ package org.utbot.instrumentation.instrumentation.execution.ndd -import org.utbot.framework.plugin.api.ClassId -import org.utbot.framework.plugin.api.MethodId -import org.utbot.framework.plugin.api.util.id -import org.utbot.framework.plugin.api.util.isStatic -import kotlin.random.Random - class NonDeterministicDetector { - private val nonDeterministicStaticMethods: HashSet = HashSet() + private val nonDeterministicStaticMethods: HashSet = HashSet() - private val nonDeterministicClasses: HashSet = buildList { - add(java.util.Random::class.java) - add(Random::class.java) - }.map { it.id }.toHashSet() + private val nonDeterministicClasses: HashSet = buildList { + add("java/util/Random") + add("kotlin/random/Random") + }.toHashSet() val inserter = NonDeterministicBytecodeInserter() - fun isNonDeterministic(caller: ClassId, method: MethodId): Boolean { - return if (method.isStatic) { - nonDeterministicStaticMethods.contains(method) - } else { - isNonDeterministic(caller) - } + fun isNonDeterministicStaticFunction(owner: String, name: String, descriptor: String): Boolean { + return nonDeterministicStaticMethods.contains("$owner $name$descriptor") } - fun isNonDeterministic(clazz: ClassId): Boolean { - var cl: ClassId? = clazz - while (cl != null && !nonDeterministicClasses.contains(cl)) { - cl = cl.superclass?.id - } - return nonDeterministicClasses.contains(cl) + fun isNonDeterministicClass(clazz: String): Boolean { + return nonDeterministicClasses.contains(clazz) } } \ No newline at end of file diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicResultStorage.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicResultStorage.kt index e3683c3d26..da612f63cc 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicResultStorage.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/ndd/NonDeterministicResultStorage.kt @@ -28,8 +28,8 @@ object NonDeterministicResultStorage { nextInstanceNumber = 1 } - fun methodToSignature(methodId: MethodId): String { - return "${methodId.classId.name} ${methodId.signature}" + fun makeSignature(owner: String, name: String, descriptor: String): String { + return "$owner $name$descriptor" } fun signatureToMethod(signature: String): MethodId? {