@article{HutzelmannMaukschPetrovskaetal.2023, author = {Hutzelmann, Thomas and Mauksch, Dominik and Petrovska, Ana and Pretschner, Alexander}, title = {Generation of Tailored and Confined Datasets for IDS Evaluation in Cyber-Physical Systems}, volume = {21}, journal = {IEEE Transactions on Dependable and Secure Computing}, number = {4}, publisher = {IEEE}, address = {New York}, issn = {1941-0018}, doi = {https://doi.rog/10.1109/TDSC.2023.3341211}, pages = {3948 -- 3962}, year = {2023}, abstract = {The state-of-the-art evaluation of an Intrusion Detection System (IDS) relies on benchmark datasets composed of the regular system's and potential attackers' behavior. The datasets are collected once and independently of the IDS under analysis. This paper questions this practice by introducing a methodology to elicit particularly challenging samples to benchmark a given IDS. In detail, we propose (1) six fitness functions quantifying the suitability of individual samples, particularly tailored for safety-critical cyber-physical systems, (2) a scenario-based methodology for attacks on networks to systematically deduce optimal samples in addition to previous datasets, and (3) a respective extension of the standard IDS evaluation methodology. We applied our methodology to two network-based IDSs defending an advanced driver assistance system. Our results indicate that different IDSs show strongly differing characteristics in their edge case classifications and that the original datasets used for evaluation do not include such challenging behavior. In the worst case, this causes a critical undetected attack, as we document for one IDS. Our findings highlight the need to tailor benchmark datasets to the individual IDS in a final evaluation step. Especially the manual investigation of selected samples from edge case classifications by domain experts is vital for assessing the IDSs.}, language = {en} } @article{PetrovskaKugeleHutzelmannetal.2022, author = {Petrovska, Ana and Kugele, Stefan and Hutzelmann, Thomas and Beffart, Theo and Bergemann, Sebastian and Pretschner, Alexander}, title = {Defining adaptivity and logical architecture for engineering (smart) self-adaptive cyber-physical systems}, volume = {2022}, pages = {106866}, journal = {Information and Software Technology}, number = {147}, publisher = {Elsevier}, address = {Amsterdam}, issn = {0950-5849}, doi = {https://doi.org/10.1016/j.infsof.2022.106866}, year = {2022}, language = {en} } @inproceedings{ZieglmeierKaciankaHutzelmannetal.2019, author = {Zieglmeier, Valentin and Kacianka, Severin and Hutzelmann, Thomas and Pretschner, Alexander}, title = {A Real-Time Remote IDS Testbed for Connected Vehicles}, booktitle = {Proceedings of the 34th ACM/SIGAPP Symposium on Applied Computing}, publisher = {ACM}, address = {New York}, isbn = {978-1-4503-5933-7}, doi = {https://doi.org/10.1145/3297280.3297465}, pages = {1898 -- 1905}, year = {2019}, language = {en} } @inproceedings{LippElsnerHutzelmannetal.2022, author = {Lipp, Stephan and Elsner, Daniel and Hutzelmann, Thomas and Banescu, Sebastian and Pretschner, Alexander and B{\"o}hme, Marcel}, title = {FuzzTastic: A Fine-grained, Fuzzer-agnostic Coverage Analyzer}, booktitle = {Proceedings, 2022 ACM/IEEE 44th International Conference on Software Engineering: Companion Proceedings}, publisher = {ACM}, address = {New York}, issn = {978-1-4503-9223-5}, doi = {https://doi.org/10.1145/3510454.3516847}, pages = {75 -- 79}, year = {2022}, abstract = {Performing sound and fair fuzzer evaluations can be challenging, not only because of the randomness involved in fuzzing, but also due to the large number of fuzz tests generated. Existing evaluations use code coverage as a proxy measure for fuzzing effectiveness. Yet, instead of considering coverage of all generated fuzz inputs, they only consider the inputs stored in the fuzzer queue. However, as we show in this paper, this approach can lead to biased assessments due to path collisions. Therefore, we developed FuzzTastic, a fuzzeragnostic coverage analyzer that allows practitioners and researchers to perform uniform fuzzer evaluations that are not affected by such collisions. In addition, its time-stamped coverage-probing approach enables frequency-based coverage analysis to identify barely tested source code and to visualize fuzzing progress over time and across code. To foster further studies in this field, we make FuzzTastic, together with a benchmark dataset worth ~12 CPU-years of fuzzing, publicly available; the demo video can be found at https://youtu.be/Lm-eBx0aePA.}, language = {en} } @article{HutzelmannBanescuPretschner2019, author = {Hutzelmann, Thomas and Banescu, Sebastian and Pretschner, Alexander}, title = {A Comprehensive Attack and Defense Model for the Automotive Domain}, volume = {2}, pages = {11-02-01-0001}, journal = {SAE International Journal of Transportation Cybersecurity and Privacy}, number = {1}, publisher = {SAE International}, address = {Warrendale}, issn = {2572-1054}, doi = {https://doi.org/10.4271/11-02-01-0001}, year = {2019}, language = {en} } @unpublished{OgnawalaPretschnerHutzelmannetal.2018, author = {Ognawala, Saahil and Pretschner, Alexander and Hutzelmann, Thomas and Psallida, Eirini and Amato, Ricardo Nales}, title = {Reviewing KLEE's Sonar-Search Strategy in Context of Greybox Fuzzing}, publisher = {arXiv}, address = {Ithaca}, doi = {https://doi.org/10.48550/arXiv.1803.04881}, year = {2018}, abstract = {Automatic test-case generation techniques of symbolic execution and fuzzing are the most widely used methods to discover vulnerabilities in, both, academia and industry. However, both these methods suffer from fundamental drawbacks that stop them from achieving high path coverage that may, consequently, lead to discovering vulnerabilities at the numerical scale of static analysis. In this presentation, we examine systems-under-test (SUTs) at the granularity level of functions and postulate that achieving higher function coverage (execution of functions in a program at least once) than, both, symbolic execution and fuzzing may be a necessary condition for discovering more vulnerabilities than both. We will start this presentation with the design of a targeted search strategy for KLEE, sonar-search, that prioritizes paths leading to a target function, rather than maximizing overall path coverage in the program. Then, we will show that examining SUTs at the level of functions (compositional analysis) leads to discovering more vulnerabilities than symbolic execution from a single entry point. Using this finding, we will, then, demonstrate a greybox fuzzing method that can achieve higher function coverage than symbolic execution. Finally, we will present a framework to effectively manage vulnerabilities and assess their severities.}, language = {en} } @inproceedings{OgnawalaHutzelmannPsallidaetal.2018, author = {Ognawala, Saahil and Hutzelmann, Thomas and Psallida, Eirini and Pretschner, Alexander}, title = {Improving Function Coverage with Munch: A Hybrid Fuzzing and Directed Symbolic Execution Approach}, booktitle = {Proceedings of the 33rd Annual ACM Symposium on Applied Computing}, publisher = {ACM}, address = {New York}, isbn = {978-1-4503-5191-1}, doi = {https://doi.org/10.1145/3167132.3167289}, pages = {1475 -- 1482}, year = {2018}, language = {en} } @inproceedings{HutzelmannMaukschPretschner2020, author = {Hutzelmann, Thomas and Mauksch, Dominik and Pretschner, Alexander}, title = {How to Conduct Experiments with a Real Car? Experiences and Practical Guidelines}, booktitle = {Software Architecture: 14th European Conference, ECSA 2020, Tracks and Workshops, L'Aquila, Italy, September 14-18, 2020, Proceedings}, editor = {Muccini, Henry and Avgeriou, Paris and Buhnova, Barbora and Camara, Javier and Caporuscio, Mauro and Franzago, Mirco and Koziolek, Anne and Scandurra, Patrizia and Trubiani, Catia and Weyns, Danny and Zdun, Uwe}, publisher = {Springer}, address = {Cham}, isbn = {978-3-030-59155-7}, doi = {https://doi.org/10.1007/978-3-030-59155-7_37}, pages = {518 -- 526}, year = {2020}, language = {en} }