Improve modificators analysis in UtBotFieldModificatorsSearcher and AssembleModelGenerator

Author: EgorkaKulikov

utbot-framework-test/src/main/java/org/utbot/examples/assemble/DefaultPackagePrivateField.java

@@ -0,0 +1,9 @@
+package org.utbot.examples.assemble;
+
+/**
+ * Need to be located at the same package as [AssembleTestUtils]
+ * because requires a setter for package-private field.
+ */
+public class DefaultPackagePrivateField {
+    int z = 10;
+}

utbot-framework-test/src/main/java/org/utbot/examples/assemble/defaults/DefaultFieldModifiedInConstructor.java

@@ -1,7 +1,7 @@
 package org.utbot.examples.assemble.defaults;
 
 public class DefaultFieldModifiedInConstructor {
-    int z;
+    public int z;
 
     @SuppressWarnings("Unused")
     DefaultFieldModifiedInConstructor(int z_) {

utbot-framework-test/src/main/java/org/utbot/examples/assemble/defaults/DefaultPackagePrivateField.java

@@ -35,7 +35,7 @@ import org.utbot.examples.assemble.defaults.DefaultField
 import org.utbot.examples.assemble.defaults.DefaultFieldModifiedInConstructor
 import org.utbot.examples.assemble.defaults.DefaultFieldWithDirectAccessor
 import org.utbot.examples.assemble.defaults.DefaultFieldWithSetter
-import org.utbot.examples.assemble.defaults.DefaultPackagePrivateField
+import org.utbot.examples.assemble.DefaultPackagePrivateField
 import org.utbot.examples.assemble.statics.StaticField
 import org.utbot.framework.plugin.api.ClassId
 import org.utbot.framework.plugin.api.ExecutableId
@@ -58,7 +58,6 @@ import org.utbot.framework.util.SootUtils
 import org.utbot.framework.util.instanceCounter
 import org.utbot.framework.util.modelIdCounter
 import kotlin.reflect.full.functions
-import org.utbot.examples.assemble.*
 import org.utbot.framework.codegen.model.constructor.util.arrayTypeOf
 
 /**
@@ -150,8 +149,7 @@ class AssembleModelGeneratorTests {
             fields(testClassId, "a" to 5, "b" to 3)
         )
 
-        val methodFromAnotherPackage =
-            MethodUnderTest::class.functions.first()
+        val methodFromAnotherPackage = MethodUnderTest::class.functions.first()
 
         createModelAndAssert(compositeModel, null, methodFromAnotherPackage.executableId)
     }
@@ -413,7 +411,7 @@ class AssembleModelGeneratorTests {
         val baseClassId = PrimitiveFields::class.id
 
         val thisFields = fields(inheritedFieldClassId, "i" to 5, "d" to 3.0)
-        val baseFields = fields(baseClassId, "a" to 2, "b" to 4)
+        val baseFields = fields(baseClassId, "b" to 4)
 
         val compositeModel = UtCompositeModel(
             modelIdCounter.incrementAndGet(),
@@ -425,7 +423,6 @@ class AssembleModelGeneratorTests {
         val v1 = statementsChain.addExpectedVariableDecl<InheritedField>()
         statementsChain.add("$v1." + ("i" `=` 5))
         statementsChain.add("$v1." + ("d" `=` 3.0))
-        statementsChain.add("$v1." + addExpectedSetter("a", 2))
         statementsChain.add("$v1." + ("b" `=` 4))
 
         val expectedRepresentation = printExpectedModel(inheritedFieldClassId.simpleName, v1, statementsChain)
@@ -1448,9 +1445,9 @@ class AssembleModelGeneratorTests {
     private fun createModelsAndAssert(
         models: List<UtModel>,
         expectedModelRepresentations: List<String?>,
-        assembleTestUtils: ExecutableId = AssembleTestUtils::class.id.allMethods.first(),
+        assembleTestDummyMethod: ExecutableId = AssembleTestUtils::class.id.allMethods.first(),
     ) {
-        val modelsMap = AssembleModelGenerator(assembleTestUtils.classId.packageName).createAssembleModels(models)
+        val modelsMap = AssembleModelGenerator(assembleTestDummyMethod.classId.packageName).createAssembleModels(models)
         //we sort values to fix order of models somehow (IdentityHashMap does not guarantee the order)
         val assembleModels = modelsMap.values
             .filterIsInstance<UtAssembleModel>()

utbot-framework-test/src/test/kotlin/org/utbot/framework/assemble/AssembleModelGeneratorTests.kt

@@ -192,7 +192,7 @@ internal class UtBotFieldModificatorsTest {
 
     //We use sorting here to make comparing with sorted in advance expected collections easier
     private fun runFieldModificatorsSearch(analysisMode: AnalysisMode) =
-        fieldsModificatorsSearcher.findModificators(analysisMode, PrimitiveModifications::class.java.packageName)
+        fieldsModificatorsSearcher.findModificators(analysisMode)
             .map { (key, value) ->
                 val modificatorNames = value.filterNot { it.name.startsWith("direct_set_") }.map { it.name }
                 key.name to modificatorNames.toSortedSet()

utbot-framework-test/src/test/kotlin/org/utbot/framework/modificators/UtBotFieldModificatorsTest.kt

@@ -8,6 +8,7 @@ import org.utbot.framework.plugin.api.UtArrayModel
 import org.utbot.framework.plugin.api.UtClassRefModel
 import org.utbot.framework.plugin.api.UtCompositeModel
 import org.utbot.framework.plugin.api.UtModel
+import org.utbot.framework.plugin.api.util.executableId
 import org.utbot.framework.plugin.api.util.id
 import java.lang.reflect.Field
 import java.lang.reflect.Method

utbot-framework/src/main/kotlin/org/utbot/external/api/UtModelFactory.kt

@@ -36,20 +36,21 @@ import org.utbot.framework.plugin.api.hasDefaultValue
 import org.utbot.framework.plugin.api.isMockModel
 import org.utbot.framework.plugin.api.util.defaultValueModel
 import org.utbot.framework.plugin.api.util.executableId
+import org.utbot.framework.plugin.api.util.isSubtypeOf
 import org.utbot.framework.plugin.api.util.jClass
 import org.utbot.framework.util.nextModelName
 import java.lang.reflect.Constructor
 import java.util.IdentityHashMap
 
 /**
  * Creates [UtAssembleModel] from any [UtModel] or it's inner models if possible
- * during generation test for [methodUnderTest].
+ * during generation test for [basePackageName].
  *
  * Needs utContext be set and Soot be initialized.
  *
  * Note: Caches class related information, can be reused if classes don't change.
  */
-class AssembleModelGenerator(private val methodPackageName: String) {
+class AssembleModelGenerator(private val basePackageName: String) {
 
     //Instantiated models are stored to avoid cyclic references during reference graph analysis
     private val instantiatedModels: IdentityHashMap<UtModel, UtReferenceModel> =
@@ -171,7 +172,7 @@ class AssembleModelGenerator(private val methodPackageName: String) {
     private fun assembleModel(utModel: UtModel): UtModel {
         val collectedCallChain = callChain.toMutableList()
 
-        // we cannot create an assemble model for an anonymous class instance
+        // We cannot create an assemble model for an anonymous class instance
         if (utModel.classId.isAnonymous) {
             return utModel
         }
@@ -256,7 +257,7 @@ class AssembleModelGenerator(private val methodPackageName: String) {
                     if (fieldId.isFinal) {
                         throw AssembleException("Final field $fieldId can't be set in an object of the class $classId")
                     }
-                    if (!fieldId.type.isAccessibleFrom(methodPackageName)) {
+                    if (!fieldId.type.isAccessibleFrom(basePackageName)) {
                         throw AssembleException(
                             "Field $fieldId can't be set in an object of the class $classId because its type is inaccessible"
                         )
@@ -265,7 +266,8 @@ class AssembleModelGenerator(private val methodPackageName: String) {
                     if (fieldId in constructorInfo.affectedFields ||
                             (fieldId !in constructorInfo.setFields && !fieldModel.hasDefaultValue())
                     ) {
-                        val modifierCall = modifierCall(this, fieldId, assembleModel(fieldModel))
+                        val assembledModel = assembleModel(fieldModel)
+                        val modifierCall = modifierCall(this, fieldId, assembledModel)
                         callChain.add(modifierCall)
                     }
                 }
@@ -397,10 +399,10 @@ class AssembleModelGenerator(private val methodPackageName: String) {
     }
 
     private val ClassId.isVisible : Boolean
-        get() = this.isPublic || !this.isPrivate && this.packageName.startsWith(methodPackageName)
+        get() = this.isPublic || !this.isPrivate && this.packageName.startsWith(basePackageName)
 
     private val Constructor<*>.isVisible : Boolean
-        get() = this.isPublic || !this.isPrivate && this.declaringClass.packageName.startsWith(methodPackageName)
+        get() = this.isPublic || !this.isPrivate && this.declaringClass.packageName.startsWith(basePackageName)
 
     /**
      * Creates setter or direct setter call to set a field.
@@ -414,7 +416,7 @@ class AssembleModelGenerator(private val methodPackageName: String) {
     ): UtStatementModel {
         val declaringClassId = fieldId.declaringClass
 
-        val modifiers = getOrFindSettersAndDirectAccessors(declaringClassId)
+        val modifiers = getOrFindSettersAndDirectAccessors(instance.classId)
         val modifier = modifiers[fieldId]
             ?: throw AssembleException("No setter for field ${fieldId.name} of class ${declaringClassId.name}")
 
@@ -439,9 +441,7 @@ class AssembleModelGenerator(private val methodPackageName: String) {
      * Finds setters and direct accessors for fields of particular class.
      */
     private fun findSettersAndDirectAccessors(classId: ClassId): Map<FieldId, StatementId> {
-        val allModificatorsOfClass =  modificatorsSearcher
-            .findModificators(SettersAndDirectAccessors, methodPackageName)
-            .map { it.key to it.value.filter { st -> st.classId == classId } }
+        val allModificatorsOfClass =  modificatorsSearcher.findModificators(SettersAndDirectAccessors)
 
         return allModificatorsOfClass
             .mapNotNull { (fieldId, possibleModificators) ->
@@ -457,9 +457,12 @@ class AssembleModelGenerator(private val methodPackageName: String) {
      */
     private fun chooseModificator(
         fieldId: FieldId,
-        settersAndDirectAccessors: List<StatementId>
+        settersAndDirectAccessors: Set<StatementId>,
     ): StatementId? {
-        val directAccessors = settersAndDirectAccessors.filterIsInstance<DirectFieldAccessId>()
+        val directAccessors = settersAndDirectAccessors
+            .filterIsInstance<DirectFieldAccessId>()
+            .filter {it.fieldId.isAccessibleFrom(basePackageName) }
+
         if (directAccessors.any()) {
             return directAccessors.singleOrNull()
                 ?: throw AssembleException(
@@ -468,7 +471,9 @@ class AssembleModelGenerator(private val methodPackageName: String) {
         }
 
         if (settersAndDirectAccessors.any()) {
-            return settersAndDirectAccessors.singleOrNull()
+            return settersAndDirectAccessors
+                .filterIsInstance<ExecutableId>()
+                .singleOrNull { it.isAccessibleFrom(basePackageName) }
                 ?: throw AssembleException(
                     "Field $fieldId has more than one setter: ${settersAndDirectAccessors.joinToString(" ")}"
                 )

utbot-framework/src/main/kotlin/org/utbot/framework/assemble/AssembleModelGenerator.kt

@@ -129,6 +129,10 @@ internal class CgCallableAccessManagerImpl(val context: CgContext) : CgCallableA
                 )
             }
             // we can access the field only via reflection
+
+            // NOTE that current implementation works only if field access is located
+            // in the right part of the assignment. However, obtaining this construction
+            // as an "l-value" seems to be an error in assemble models or somewhere else.
             is ReflectionOnly -> fieldId.accessWithReflection(this)
         }
     }

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgCallableAccessManager.kt

@@ -13,9 +13,7 @@ import org.utbot.framework.plugin.api.util.voidClassId
  *
  * @param context context in which code is generated (it is needed because the method needs to know package and language)
  */
-// TODO: change parameter from packageName: String to context: CgContext in ClassId.isAccessibleFrom and ExecutableId.isAccessibleFrom ?
-private fun FieldId.isAccessibleFrom(context: CgContext): Boolean {
-    val packageName = context.testClassPackageName
+fun FieldId.isAccessibleFrom(packageName: String): Boolean {
     val isClassAccessible = declaringClass.isAccessibleFrom(packageName)
     val isAccessibleByVisibility = isPublic || (declaringClass.packageName == packageName && (isPackagePrivate || isProtected))
     val isAccessibleFromPackageByModifiers = isAccessibleByVisibility && !isSynthetic
@@ -36,7 +34,7 @@ internal infix fun FieldId.canBeReadFrom(context: CgContext): Boolean {
             return true
     }
 
-    return isAccessibleFrom(context)
+    return isAccessibleFrom(context.testClassPackageName)
 }
 
 private fun FieldId.canBeSetViaSetterFrom(context: CgContext): Boolean =
@@ -49,12 +47,12 @@ internal fun FieldId.canBeSetFrom(context: CgContext): Boolean {
     if (context.codegenLanguage == CodegenLanguage.KOTLIN) {
         // Kotlin will allow direct write access if both getter and setter is defined
         // !isAccessibleFrom(context) is important here because above rule applies to final fields only if they are not accessible in Java terms
-        if (!isAccessibleFrom(context) && !isStatic && canBeReadViaGetterFrom(context) && canBeSetViaSetterFrom(context)) {
+        if (!isAccessibleFrom(context.testClassPackageName) && !isStatic && canBeReadViaGetterFrom(context) && canBeSetViaSetterFrom(context)) {
             return true
         }
     }
 
-    return isAccessibleFrom(context) && !isFinal
+    return isAccessibleFrom(context.testClassPackageName) && !isFinal
 }
 
 /**

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/util/FieldIdUtil.kt

@@ -8,6 +8,7 @@ import org.utbot.framework.UtSettings
 import org.utbot.framework.assemble.AssembleModelGenerator
 import org.utbot.framework.plugin.api.Coverage
 import org.utbot.framework.plugin.api.EnvironmentModels
+import org.utbot.framework.plugin.api.ExecutableId
 import org.utbot.framework.plugin.api.FieldId
 import org.utbot.framework.plugin.api.Instruction
 import org.utbot.framework.plugin.api.MissingState
@@ -98,9 +99,7 @@ class UtConcreteExecutionResult(
      *
      * @return [UtConcreteExecutionResult] with converted models.
      */
-    fun convertToAssemble(
-        packageName: String
-    ): UtConcreteExecutionResult {
+    fun convertToAssemble(packageName: String): UtConcreteExecutionResult {
         val allModels = collectAllModels()
 
         val modelsToAssembleModels = AssembleModelGenerator(packageName).createAssembleModels(allModels)

utbot-framework/src/main/kotlin/org/utbot/framework/concrete/UtExecutionInstrumentation.kt

@@ -17,19 +17,24 @@ class DirectAccessorsAnalyzer {
      */
     fun collectDirectAccesses(classIds: Set<ClassId>): Set<DirectFieldAccessId> =
         classIds
-            .flatMap { classId -> collectFieldsInPackage(classId) }
+            .flatMap { classId -> collectFields(classId) }
             .map { fieldId -> DirectFieldAccessId(fieldId.declaringClass, directSetterName(fieldId), fieldId) }
             .toSet()
 
     /**
-     * Collect all fields with different non-private modifiers from class [classId].
+     * Collect all fields with different non-private modifiers
+     * from class [classId] or it's base classes.
      */
-    private fun collectFieldsInPackage(classId: ClassId): Set<FieldId> {
-        val clazz = classId.jClass
+    private fun collectFields(classId: ClassId): Set<FieldId> {
+        var clazz = classId.jClass
 
         val fieldIds = mutableSetOf<Field>()
-        fieldIds += clazz.fields
         fieldIds += clazz.declaredFields.filterNot { Modifier.isPrivate(it.modifiers) }
+        while (clazz.superclass != null) {
+            clazz = clazz.superclass
+            fieldIds += clazz.declaredFields.filterNot { Modifier.isPrivate(it.modifiers) }
+        }
+
 
         return fieldIds.map { it.fieldId }.toSet()
     }

utbot-framework/src/main/kotlin/org/utbot/framework/modifications/DirectAccessorsAnalyzer.kt

@@ -1,8 +1,10 @@
 package org.utbot.framework.modifications
 
 import org.utbot.framework.plugin.api.ClassId
+import org.utbot.framework.plugin.api.ExecutableId
 import org.utbot.framework.plugin.api.FieldId
 import org.utbot.framework.plugin.api.StatementId
+import org.utbot.framework.plugin.api.util.isSubtypeOf
 
 class UtBotFieldsModificatorsSearcher {
 
@@ -12,28 +14,7 @@ class UtBotFieldsModificatorsSearcher {
 
     fun delete(classIds: Set<ClassId>) = statementsStorage.delete(classIds)
 
-    /**
-     * Finds field modificators.
-     *
-     * @param analysisMode represents which type of modificators (e.g. setters) are considered.
-     * @param packageName describes a location of package-private methods that need to be considered.
-     */
-    fun findModificators(analysisMode: AnalysisMode, packageName: String? = null): Map<FieldId, Set<StatementId>> {
-        val modificators = findModificators(analysisMode)
-        if (packageName == null) {
-            return modificators
-        }
-
-        val filteredModifications = mutableMapOf<FieldId, Set<StatementId>>()
-        for ((fieldId, statements) in modificators) {
-            val filteredStmts = statements.filter { it.classId.packageName.startsWith(packageName) }.toSet()
-            filteredModifications[fieldId] = filteredStmts
-        }
-
-        return filteredModifications
-    }
-
-    private fun findModificators(analysisMode: AnalysisMode): Map<FieldId, Set<StatementId>> {
+    fun findModificators(analysisMode: AnalysisMode): Map<FieldId, Set<StatementId>> {
         statementsStorage.updateCaches()
         return findModificatorsInCache(analysisMode)
     }