@article{HutzelmannMaukschPetrovskaetal.2023, author = {Hutzelmann, Thomas and Mauksch, Dominik and Petrovska, Ana and Pretschner, Alexander}, title = {Generation of Tailored and Confined Datasets for IDS Evaluation in Cyber-Physical Systems}, volume = {21}, journal = {IEEE Transactions on Dependable and Secure Computing}, number = {4}, publisher = {IEEE}, address = {New York}, issn = {1941-0018}, doi = {https://doi.rog/10.1109/TDSC.2023.3341211}, pages = {3948 -- 3962}, year = {2023}, abstract = {The state-of-the-art evaluation of an Intrusion Detection System (IDS) relies on benchmark datasets composed of the regular system's and potential attackers' behavior. The datasets are collected once and independently of the IDS under analysis. This paper questions this practice by introducing a methodology to elicit particularly challenging samples to benchmark a given IDS. In detail, we propose (1) six fitness functions quantifying the suitability of individual samples, particularly tailored for safety-critical cyber-physical systems, (2) a scenario-based methodology for attacks on networks to systematically deduce optimal samples in addition to previous datasets, and (3) a respective extension of the standard IDS evaluation methodology. We applied our methodology to two network-based IDSs defending an advanced driver assistance system. Our results indicate that different IDSs show strongly differing characteristics in their edge case classifications and that the original datasets used for evaluation do not include such challenging behavior. In the worst case, this causes a critical undetected attack, as we document for one IDS. Our findings highlight the need to tailor benchmark datasets to the individual IDS in a final evaluation step. Especially the manual investigation of selected samples from edge case classifications by domain experts is vital for assessing the IDSs.}, language = {en} } @article{PetrovskaKugeleHutzelmannetal.2022, author = {Petrovska, Ana and Kugele, Stefan and Hutzelmann, Thomas and Beffart, Theo and Bergemann, Sebastian and Pretschner, Alexander}, title = {Defining adaptivity and logical architecture for engineering (smart) self-adaptive cyber-physical systems}, volume = {2022}, pages = {106866}, journal = {Information and Software Technology}, number = {147}, publisher = {Elsevier}, address = {Amsterdam}, issn = {0950-5849}, doi = {https://doi.org/10.1016/j.infsof.2022.106866}, year = {2022}, language = {en} } @inproceedings{PetrovskaHutzelmannKugele2023, author = {Petrovska, Ana and Hutzelmann, Thomas and Kugele, Stefan}, title = {A Theoretical Framework for Self-Adaptive Systems: Specifications, Formalisation, and Architectural Implications}, booktitle = {SAC '23: Proceedings of the 38th ACM/SIGAPP Symposium on Applied Computing}, publisher = {ACM}, address = {New York}, isbn = {978-1-4503-9517-5}, doi = {https://doi.org/10.1145/3555776.3577665}, pages = {1440 -- 1449}, year = {2023}, language = {en} }