@article{DeinerFeldmeierFraseretal.2023, author = {Deiner, Adina and Feldmeier, Patric and Fraser, Gordon and Schweikl, Sebastian and Wang, Wengran}, title = {Automated test generation for SCRATCH programs}, series = {Empirical Software Engineering}, volume = {28}, journal = {Empirical Software Engineering}, number = {3}, publisher = {Springer Nature}, address = {Berlin}, doi = {10.1007/s10664-022-10255-x}, url = {http://nbn-resolving.de/urn:nbn:de:101:1-2023091108301581209964}, pages = {1 -- 63}, year = {2023}, abstract = {The importance of programming education has led to dedicated educational program- ming environments, where users visually arrange block-based programming constructs that typically control graphical, interactive game-like programs. The SCRATCH programming environment is particularly popular, with more than 90 million registered users at the time of this writing. While the block-based nature of S CRATCH helps learners by preventing syntactical mistakes, there nevertheless remains a need to provide feedback and support in order to implement desired functionality. To support individual learning and classroom settings, this feedback and support should ideally be provided in an automated fashion, which requires tests to enable dynamic program analysis. In prior work we introduced W HISKER , a framework that enables automated testing of S CRATCH programs. However, creating these automated tests for S CRATCH programs is challenging. In this paper, we therefore investigate how to automatically generate W HISKER tests. Generating tests for S CRATCH raises important challenges: First, game-like programs are typically randomised, leading to flaky tests. Second, S CRATCH programs usually consist of animations and interactions with long delays, inhibiting the application of classical test generation approaches. Thus, the new application domain raises the question of which test generation technique is best suited to produce high coverage tests capable of detecting faulty behaviour. We investigate these questions using an extension of the W HISKER test framework for automated test generation. Evaluation on common programming exercises, a random sample of 1000 S CRATCH user programs, and the 1000 most popular S CRATCH programs demonstrates that our approach enables W HISKER to reliably accelerate test executions, and even though many SCRATCH programs are small and easy to cover, there are many unique challenges for which advanced search-based test generation using many-objective algorithms is needed in order to achieve high coverage.}, language = {en} }