Python codegen improvement: timeouted tests and exceptions (#2101)

tamarinvs19 · web-flow · commit e6e39d38c521 · 2023-04-12T00:46:11.000+03:00
* Add skipping of tests with too long execution

* Add asserts for exception

* Add timeouted tests limit and fix setting fail/pass exceptions

* Fix handleTimeoutResult

* Fix pytest skip decorator

* Fix mistake in pytest error message
diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/tree/CgMethodConstructor.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/tree/CgMethodConstructor.kt
@@ -418,7 +418,7 @@ open class CgMethodConstructor(val context: CgContext) : CgContextOwner by conte
         return { +actual[streamConsumingMethodId]() }
     }
 
-    protected fun shouldTestPassWithException(execution: UtExecution, exception: Throwable): Boolean {
+    protected open fun shouldTestPassWithException(execution: UtExecution, exception: Throwable): Boolean {
         if (exception is AccessControlException) return false
         // tests with timeout or crash should be processed differently
         if (exception is TimeoutException || exception is InstrumentedProcessDeathException) return false
diff --git a/utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/language/python/PythonDialogProcessor.kt b/utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/language/python/PythonDialogProcessor.kt
@@ -205,6 +205,7 @@ object PythonDialogProcessor {
                             val message = it.fold(StringBuilder()) { acc, line -> acc.appendHtmlLine(line) }
                             WarningTestsReportNotifier.notify(message.toString())
                         },
+                        runtimeExceptionTestsBehaviour = model.runtimeExceptionTestsBehaviour,
                         startedCleaningAction = { indicator.text = "Cleaning up..." }
                     )
                 } finally {
diff --git a/utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/language/python/PythonDialogWindow.kt b/utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/language/python/PythonDialogWindow.kt
@@ -177,6 +177,7 @@ class PythonDialogWindow(val model: PythonTestsModel) : DialogWrapper(model.proj
         val settings = model.project.service<Settings>()
         with(settings) {
             model.timeoutForRun = hangingTestsTimeout.timeoutMs
+            model.runtimeExceptionTestsBehaviour = runtimeExceptionTestsBehaviour
         }
 
         super.doOKAction()
diff --git a/utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/language/python/PythonTestsModel.kt b/utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/language/python/PythonTestsModel.kt
@@ -5,6 +5,7 @@ import com.intellij.openapi.project.Project
 import com.jetbrains.python.psi.PyClass
 import com.jetbrains.python.psi.PyFile
 import com.jetbrains.python.psi.PyFunction
+import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour
 import org.utbot.framework.codegen.domain.TestFramework
 import org.utbot.framework.codegen.services.language.CgLanguageAssistant
 import org.utbot.intellij.plugin.models.BaseTestsModel
@@ -31,4 +32,5 @@ class PythonTestsModel(
     lateinit var testSourceRootPath: String
     lateinit var testFramework: TestFramework
     lateinit var selectedFunctions: Set<PyFunction>
+    lateinit var runtimeExceptionTestsBehaviour: RuntimeExceptionTestsBehaviour
 }
diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt
@@ -47,7 +47,7 @@ class PythonEngine(
                 // can be improved
                 description.name
             }
-            is UtExplicitlyThrownException -> "${description.name}_with_exception"
+            is UtExecutionFailure -> "${description.name}_with_exception"
             else -> description.name
         }
         val testName = "test_$testSuffix"
@@ -81,6 +81,37 @@ class PythonEngine(
         return Pair(stateThisObject, modelList)
     }
 
+    private fun handleTimeoutResult(
+        arguments: List<PythonFuzzedValue>,
+        methodUnderTestDescription: PythonMethodDescription,
+    ): FuzzingExecutionFeedback {
+        val summary = arguments
+            .zip(methodUnderTest.arguments)
+            .mapNotNull { it.first.summary?.replace("%var%", it.second.name) }
+        val executionResult = UtTimeoutException(TimeoutException("Execution is too long"))
+        val testMethodName = suggestExecutionName(methodUnderTestDescription, executionResult)
+
+        val hasThisObject = methodUnderTest.hasThisArgument
+        val (beforeThisObjectTree, beforeModelListTree) = if (hasThisObject) {
+            arguments.first() to arguments.drop(1)
+        } else {
+            null to arguments
+        }
+        val beforeThisObject = beforeThisObjectTree?.let { PythonTreeModel(it.tree) }
+        val beforeModelList = beforeModelListTree.map { PythonTreeModel(it.tree) }
+
+        val utFuzzedExecution = PythonUtExecution(
+            stateInit = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap()),
+            stateBefore = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap()),
+            stateAfter = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap()),
+            diffIds = emptyList(),
+            result = executionResult,
+            testMethodName = testMethodName.testName?.camelToSnakeCase(),
+            displayName = testMethodName.displayName,
+            summary = summary.map { DocRegularStmt(it) }
+        )
+        return ValidExecution(utFuzzedExecution)
+    }
     private fun handleSuccessResult(
         arguments: List<PythonFuzzedValue>,
         types: List<Type>,
@@ -110,7 +141,7 @@ class PythonEngine(
 
         val executionResult =
             if (evaluationResult.isException) {
-                UtExplicitlyThrownException(Throwable(resultModel.type.toString()), false)
+                UtImplicitlyThrownException(Throwable(resultModel.type.toString()), false)
             }
             else {
                 UtExecutionSuccess(PythonTreeModel(resultModel))
@@ -157,8 +188,7 @@ class PythonEngine(
                     serverSocket,
                     pythonPath,
                     until,
-                    { constructEvaluationInput(it) },
-                )
+                ) { constructEvaluationInput(it) }
             } catch (_: TimeoutException) {
                 return@flow
             }
@@ -199,8 +229,8 @@ class PythonEngine(
                         }
 
                         is PythonEvaluationTimeout -> {
-                            val utError = UtError(evaluationResult.message, Throwable())
-                            PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS))
+                            val utTimeoutException = handleTimeoutResult(arguments, description)
+                            PythonExecutionResult(utTimeoutException, PythonFeedback(control = Control.PASS))
                         }
 
                         is PythonEvaluationSuccess -> {
diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt
@@ -31,6 +31,7 @@ import java.io.File
 
 private val logger = KotlinLogging.logger {}
 private const val RANDOM_TYPE_FREQUENCY = 6
+private const val MAX_EMPTY_COVERAGE_TESTS = 5
 
 class PythonTestCaseGenerator(
     private val withMinimization: Boolean = true,
@@ -262,12 +263,15 @@ class PythonTestCaseGenerator(
         }
 
         logger.info("Collect all test executions for ${method.name}")
-        val (successfulExecutions, failedExecutions) = executions.partition { it.result is UtExecutionSuccess }
+        val (emptyCoverageExecutions, coverageExecutions) = executions.partition { it.coverage == null }
+        val (successfulExecutions, failedExecutions) = coverageExecutions.partition { it.result is UtExecutionSuccess }
 
         return PythonTestSet(
             method,
             if (withMinimization)
-                minimizeExecutions(successfulExecutions) + minimizeExecutions(failedExecutions)
+                minimizeExecutions(successfulExecutions) +
+                        minimizeExecutions(failedExecutions) +
+                        emptyCoverageExecutions.take(MAX_EMPTY_COVERAGE_TESTS)
             else
                 executions,
             errors,
diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt
@@ -3,6 +3,7 @@ package org.utbot.python
 import com.squareup.moshi.Moshi
 import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory
 import org.parsers.python.PythonParser
+import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour
 import org.utbot.python.framework.codegen.model.PythonSysPathImport
 import org.utbot.python.framework.codegen.model.PythonSystemImport
 import org.utbot.python.framework.codegen.model.PythonUserImport
@@ -50,6 +51,7 @@ object PythonTestGenerationProcessor {
         pythonRunRoot: Path,
         doNotCheckRequirements: Boolean = false,
         withMinimization: Boolean = true,
+        runtimeExceptionTestsBehaviour: RuntimeExceptionTestsBehaviour = RuntimeExceptionTestsBehaviour.FAIL,
         isCanceled: () -> Boolean = { false },
         checkingRequirementsAction: () -> Unit = {},
         installingRequirementsAction: () -> Unit = {},
@@ -202,6 +204,7 @@ object PythonTestGenerationProcessor {
                     paramNames = paramNames,
                     testFramework = testFramework,
                     testClassPackageName = "",
+                    runtimeExceptionTestsBehaviour = runtimeExceptionTestsBehaviour,
                 )
                 val testCode = codegen.pythonGenerateAsStringWithTestReport(
                     notEmptyTests.map { testSet ->
diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/UtExecutorThread.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/UtExecutorThread.kt
@@ -1,8 +1,14 @@
 package org.utbot.python.evaluation
 
+import java.net.SocketException
+
 class UtExecutorThread : Thread() {
     override fun run() {
-        response = pythonWorker?.receiveMessage()
+        response = try {
+            pythonWorker?.receiveMessage()
+        } catch (ex: SocketException) {
+            null
+        }
     }
 
     enum class Status {
diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt
@@ -34,6 +34,10 @@ class PythonClassId(
     override val simpleName: String = typeName
     override val canonicalName = name
     override val packageName = moduleName
+    val prettyName: String = if (rootModuleName == pythonBuiltinsModuleName)
+        name.split(".", limit=2).last()
+    else
+        name
 }
 
 open class RawPythonAnnotation(
diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonIdUtils.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonIdUtils.kt
@@ -19,3 +19,4 @@ val pythonDictClassId = PythonClassId("builtins.dict")
 val pythonSetClassId = PythonClassId("builtins.set")
 val pythonBytearrayClassId = PythonClassId("builtins.bytearray")
 val pythonBytesClassId = PythonClassId("builtins.bytes")
+val pythonExceptionClassId = PythonClassId("builtins.Exception")
diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/PythonDomain.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/PythonDomain.kt
@@ -41,6 +41,8 @@ object Pytest : TestFramework(displayName = "pytest", id = "pytest") {
     override val nestedClassesShouldBeStatic: Boolean = false
     override val argListClassId: ClassId = pythonAnyClassId
 
+    val skipDecoratorClassId = PythonClassId("pytest", "mark.skip")
+
     @OptIn(ExperimentalStdlibApi::class)
     override fun getRunTestsCommand(
         executionInvoke: String,
@@ -89,6 +91,8 @@ object Unittest : TestFramework(displayName = "Unittest", id = "Unittest") {
     override val nestedClassesShouldBeStatic: Boolean = false
     override val argListClassId: ClassId = pythonAnyClassId
 
+    val skipDecoratorClassId = PythonClassId("unittest.skip")
+
     override fun getRunTestsCommand(
         executionInvoke: String,
         classPath: String,
diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgMethodConstructor.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgMethodConstructor.kt
@@ -1,16 +1,24 @@
 package org.utbot.python.framework.codegen.model.constructor.tree
 
+import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour
 import org.utbot.framework.codegen.domain.context.CgContext
+import org.utbot.framework.codegen.domain.models.CgDocumentationComment
 import org.utbot.framework.codegen.domain.models.CgFieldAccess
 import org.utbot.framework.codegen.domain.models.CgGetLength
 import org.utbot.framework.codegen.domain.models.CgLiteral
+import org.utbot.framework.codegen.domain.models.CgMultilineComment
+import org.utbot.framework.codegen.domain.models.CgParameterDeclaration
 import org.utbot.framework.codegen.domain.models.CgReferenceExpression
 import org.utbot.framework.codegen.domain.models.CgTestMethod
+import org.utbot.framework.codegen.domain.models.CgTestMethodType
 import org.utbot.framework.codegen.domain.models.CgValue
 import org.utbot.framework.codegen.domain.models.CgVariable
+import org.utbot.framework.codegen.domain.models.convertDocToCg
 import org.utbot.framework.codegen.tree.CgMethodConstructor
+import org.utbot.framework.codegen.tree.buildTestMethod
 import org.utbot.framework.plugin.api.*
 import org.utbot.python.framework.api.python.*
+import org.utbot.python.framework.api.python.util.pythonExceptionClassId
 import org.utbot.python.framework.api.python.util.pythonIntClassId
 import org.utbot.python.framework.api.python.util.pythonNoneClassId
 import org.utbot.python.framework.codegen.PythonCgLanguageAssistant
@@ -23,20 +31,6 @@ class PythonCgMethodConstructor(context: CgContext) : CgMethodConstructor(contex
         pythonDeepEquals(expected, actual)
     }
 
-    private fun generatePythonTestComments(execution: UtExecution) {
-        when (execution.result) {
-            is UtExplicitlyThrownException ->
-                (execution.result as UtExplicitlyThrownException).exception.message?.let {
-                    emptyLineIfNeeded()
-                    comment("raises $it")
-                }
-
-            else -> {
-                // nothing
-            }
-        }
-    }
-
     override fun createTestMethod(executableId: ExecutableId, execution: UtExecution): CgTestMethod =
         withTestMethodScope(execution) {
             val constructorState = (execution as PythonUtExecution).stateInit
@@ -48,7 +42,7 @@ class PythonCgMethodConstructor(context: CgContext) : CgMethodConstructor(contex
             }
             // TODO: remove this line when SAT-1273 is completed
             execution.displayName = execution.displayName?.let { "${executableId.name}: $it" }
-            testMethod(testMethodName, execution.displayName) {
+            pythonTestMethod(testMethodName, execution.displayName) {
                 val statics = currentExecution!!.stateBefore.statics
                 rememberInitialStaticFields(statics)
 
@@ -115,8 +109,6 @@ class PythonCgMethodConstructor(context: CgContext) : CgMethodConstructor(contex
                     generateResultAssertions()
 
                     generateFieldStateAssertions(stateAssertions, assertThisObject, executableId)
-
-                    generatePythonTestComments(execution)
                 }
 
                 if (statics.isNotEmpty()) {
@@ -131,6 +123,44 @@ class PythonCgMethodConstructor(context: CgContext) : CgMethodConstructor(contex
             }
         }
 
+    override fun generateResultAssertions() {
+        if (currentExecutable is MethodId) {
+            val currentExecution = currentExecution!!
+            val executionResult = currentExecution.result
+            if (executionResult is UtExecutionFailure) {
+                val exceptionId = executionResult.rootCauseException.message?.let {PythonClassId(it)} ?: pythonExceptionClassId
+                val executionBlock = {
+                    with(currentExecutable) {
+                        when (this) {
+                            is MethodId -> thisInstance[this](*methodArguments.toTypedArray()).intercepted()
+                            else -> {}
+                        }
+                    }
+                }
+                when (methodType) {
+                    CgTestMethodType.PASSED_EXCEPTION -> {
+                        testFrameworkManager.expectException(exceptionId) {
+                            executionBlock()
+                        }
+                        return
+                    }
+                    CgTestMethodType.FAILING -> {
+                        val executable = currentExecutable!! as PythonMethodId
+                        val executableName = "${executable.moduleName}.${executable.name}"
+                        val warningLine =
+                            "This test fails because function [$executableName] produces [${exceptionId.prettyName}]"
+                        +CgMultilineComment(warningLine)
+                        emptyLineIfNeeded()
+                        executionBlock()
+                        return
+                    }
+                    else -> {}
+                }
+            }
+        }
+        super.generateResultAssertions()
+    }
+
     private fun generateFieldStateAssertions(
         stateAssertions: MutableMap<Int, Pair<CgVariable, UtModel>>,
         assertThisObject: MutableList<Pair<CgVariable, UtModel>>,
@@ -154,6 +184,45 @@ class PythonCgMethodConstructor(context: CgContext) : CgMethodConstructor(contex
         }
     }
 
+    override fun shouldTestPassWithException(execution: UtExecution, exception: Throwable): Boolean {
+        if (exception is TimeoutException || exception is InstrumentedProcessDeathException) return false
+        return runtimeExceptionTestsBehaviour == RuntimeExceptionTestsBehaviour.PASS
+    }
+
+    private fun pythonTestMethod(
+        methodName: String,
+        displayName: String?,
+        params: List<CgParameterDeclaration> = emptyList(),
+        body: () -> Unit,
+    ): CgTestMethod {
+        displayName?.let {
+            testFrameworkManager.addTestDescription(displayName)
+        }
+
+        val result = currentExecution!!.result
+        if (result is UtTimeoutException) {
+            testFrameworkManager.disableTestMethod(
+                "Disabled due to the fact that the execution is longer then ${hangingTestsTimeout.timeoutMs} ms"
+            )
+        }
+
+        val testMethod = buildTestMethod {
+            name = methodName
+            parameters = params
+            statements = block(body)
+            // Exceptions and annotations assignment must run after the statements block is build,
+            // because we collect info about exceptions and required annotations while building the statements
+            exceptions += collectedExceptions
+            annotations += collectedMethodAnnotations
+            methodType = this@PythonCgMethodConstructor.methodType
+
+            val docComment = currentExecution?.summary?.map { convertDocToCg(it) }?.toMutableList() ?: mutableListOf()
+            documentation = CgDocumentationComment(docComment)
+        }
+        testMethods += testMethod
+        return testMethod
+    }
+
     private fun pythonDeepEquals(expected: CgValue, actual: CgVariable) {
         require(expected is CgPythonTree) {
             "Expected value have to be CgPythonTree but `${expected::class}` found"
diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonTestFrameworkManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonTestFrameworkManager.kt
diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/visitor/CgPythonRenderer.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/visitor/CgPythonRenderer.kt
diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/visitor/CgPythonVisitor.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/visitor/CgPythonVisitor.kt
diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/tree/CgPythonElement.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/tree/CgPythonElement.kt
diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt

Original file line number	Diff line number	Diff line change
`@@ -418,7 +418,7 @@ open class CgMethodConstructor(val context: CgContext) : CgContextOwner by conte`
`418`	`418`	`return { +actual[streamConsumingMethodId]() }`
`419`	`419`	`}`
`420`	`420`
`421`		`- protected fun shouldTestPassWithException(execution: UtExecution, exception: Throwable): Boolean {`
	`421`	`+ protected open fun shouldTestPassWithException(execution: UtExecution, exception: Throwable): Boolean {`
`422`	`422`	`if (exception is AccessControlException) return false`
`423`	`423`	`// tests with timeout or crash should be processed differently`
`424`	`424`	`if (exception is TimeoutException \|\| exception is InstrumentedProcessDeathException) return false`
Original file line number	Diff line number	Diff line change
`@@ -205,6 +205,7 @@ object PythonDialogProcessor {`
`205`	`205`	`val message = it.fold(StringBuilder()) { acc, line -> acc.appendHtmlLine(line) }`
`206`	`206`	`WarningTestsReportNotifier.notify(message.toString())`
`207`	`207`	`},`
	`208`	`+ runtimeExceptionTestsBehaviour = model.runtimeExceptionTestsBehaviour,`
`208`	`209`	`startedCleaningAction = { indicator.text = "Cleaning up..." }`
`209`	`210`	`)`
`210`	`211`	`} finally {`
Original file line number	Diff line number	Diff line change
`@@ -177,6 +177,7 @@ class PythonDialogWindow(val model: PythonTestsModel) : DialogWrapper(model.proj`
`177`	`177`	`val settings = model.project.service<Settings>()`
`178`	`178`	`with(settings) {`
`179`	`179`	`model.timeoutForRun = hangingTestsTimeout.timeoutMs`
	`180`	`+ model.runtimeExceptionTestsBehaviour = runtimeExceptionTestsBehaviour`
`180`	`181`	`}`
`181`	`182`
`182`	`183`	`super.doOKAction()`