@inproceedings{KnollmeyerCaymazerKovaletal.2024, author = {Knollmeyer, Simon and Caymazer, Oğuz and Koval, Leonid and Akmal, Muhammad Uzair and Asif, Saara and Mathias, Selvine George and Großmann, Daniel}, title = {Benchmarking of Retrieval Augmented Generation: A Comprehensive Systematic Literature Review on Evaluation Dimensions, Evaluation Metrics and Datasets}, booktitle = {Proceedings of the 16th International Joint Conference on Knowledge Discovery, Knowledge Engineering and Knowledge Management (IC3K 2024) - Volume 3}, editor = {Gruenwald, Le and Masciari, Elio and Bernardino, Jorge}, publisher = {SciTePress}, address = {Set{\´u}bal}, isbn = {978-989-758-716-0}, doi = {https://doi.org/10.5220/0013065700003838}, pages = {137 -- 148}, year = {2024}, abstract = {Despite the rapid advancements in the field of Large Language Models (LLM), traditional benchmarks have proven to be inadequate for assessing the performance of Retrieval Augmented Generation (RAG) systems. Therefore, this paper presents a comprehensive systematic literature review of evaluation dimensions, metrics, and datasets for RAG systems. This review identifies key evaluation dimensions such as context relevance, faithfulness, answer relevance, correctness, and citation quality. For each evaluation dimension, several metrics and evaluators are proposed on how to assess them. This paper synthesizes the findings from 12 relevant papers and presents a concept matrix that categorizes each evaluation approach. The results provide a foundation for the development of robust evaluation frameworks and suitable datasets that are essential for the effective implementation and deployment of RAG systems in real-world applications.}, language = {en} } @inproceedings{ThakurBeckMostaghimetal.2022, author = {Thakur, Akshay and Beck, Robert and Mostaghim, Sanaz and Großmann, Daniel and Kuttler, Moritz}, title = {Prediction of Top Key Performance Indicator in Automotive Production System using Data Mining}, booktitle = {WCSE 2022 Spring Event: Proceedings of 2022 9th International Conference on Industrial Engineering and Applications}, editor = {Tang, Loon-Ching}, publisher = {SCIEI}, address = {Rowland Heights}, isbn = {978-981-18-5852-9}, doi = {https://doi.org/10.18178/wcse.2022.04.079}, pages = {675 -- 684}, year = {2022}, language = {en} } @inproceedings{MuellerGrossmannSchmiedetal.2021, author = {Mueller, Ralph and Großmann, Daniel and Schmied, Sebastian and Mathias, Selvine George}, title = {Architecture of a model in the middle approach for virtual commissioning and integration of production entities}, booktitle = {Proceedings: 2021 26th IEEE International Conference on Emerging Technologies and Factory Automation (ETFA)}, publisher = {IEEE}, address = {Piscataway}, isbn = {978-1-7281-2989-1}, doi = {https://doi.org/10.1109/ETFA45728.2021.9613619}, year = {2021}, language = {en} } @inproceedings{SchmiedGrossmannMathiasetal.2021, author = {Schmied, Sebastian and Großmann, Daniel and Mathias, Selvine George and Mueller, Ralph}, title = {A concept towards the evolution and versioning of aggregated information models}, booktitle = {Proceedings: 2021 26th IEEE International Conference on Emerging Technologies and Factory Automation (ETFA)}, publisher = {IEEE}, address = {Piscataway}, isbn = {978-1-7281-2989-1}, doi = {https://doi.org/10.1109/ETFA45728.2021.9613193}, year = {2021}, language = {en} } @inproceedings{ThakurBeckMostaghimetal.2020, author = {Thakur, Akshay and Beck, Robert and Mostaghim, Sanaz and Großmann, Daniel}, title = {Survey into predictive key performance indicator analysis from data mining perspective}, booktitle = {Proceedings 2020 25th IEEE International Conference on Emerging Technologies and Factory Automation (ETFA)}, publisher = {IEEE}, address = {Piscataway}, isbn = {978-1-7281-8956-7}, doi = {https://doi.org/10.1109/ETFA46521.2020.9212111}, pages = {476 -- 483}, year = {2020}, language = {en} } @inproceedings{MathiasGrossmannSchmiedetal.2022, author = {Mathias, Selvine George and Großmann, Daniel and Schmied, Sebastian and Mueller, Ralph}, title = {Decentralized Integration Approach of Disjoint Modules in Factory Environments}, booktitle = {IEIM 2022: 2022 The 3rd International Conference on Industrial Engineering and Industrial Management}, publisher = {ACM}, address = {New York}, isbn = {978-1-4503-9569-4}, doi = {https://doi.org/10.1145/3524338.3524368}, pages = {195 -- 199}, year = {2022}, language = {en} } @inproceedings{MuellerGrossmannSchmied2022, author = {Mueller, Ralph and Großmann, Daniel and Schmied, Sebastian}, title = {Information model-based virtual commissioning for the testing and co-simulation of real production entities}, booktitle = {IEIM 2022: 2022 The 3rd International Conference on Industrial Engineering and Industrial Management}, subtitle = {Enabling of a logic for the virtual commissioning of information models using middleware technologies}, publisher = {Association for Computing Machinery}, address = {New York}, isbn = {978-1-4503-9569-4}, doi = {https://doi.org/10.1145/3524338.3524380}, pages = {271 -- 276}, year = {2022}, language = {en} } @inproceedings{KampaMuellerGrossmann2022, author = {Kampa, Thomas and M{\"u}ller, Christian Klaus and Großmann, Daniel}, title = {IP-based Architecture for an Edge Cloud enabled Factory}, booktitle = {2022 IEEE 18th International Conference on Factory Communication Systems (WFCS)}, subtitle = {Concept and Requirements}, publisher = {IEEE}, address = {Piscataway}, isbn = {978-1-6654-1086-1}, doi = {https://doi.org/10.1109/WFCS53837.2022.9779162}, year = {2022}, language = {en} } @inproceedings{AmbrosyKampaJumaretal.2022, author = {Ambrosy, Niklas and Kampa, Thomas and Jumar, Ulrich and Großmann, Daniel}, title = {5G and DetNet: Towards holistic determinism in industrial networks}, booktitle = {2022 IEEE International Conference on Industrial Technology (ICIT)}, publisher = {IEEE}, address = {Piscataway}, isbn = {978-1-72811-948-9}, doi = {https://doi.org/10.1109/ICIT48603.2022.10002776}, year = {2022}, language = {en} } @inproceedings{MirandaCabralBanerjeeetal.2017, author = {Miranda, Jorge and Cabral, Jorge and Banerjee, Suprateek and Großmann, Daniel and Fischer Pedersen, Christian and Wagner, Stefan}, title = {Analysis of OPC unified architecture for healthcare applications}, booktitle = {ETFA'2017: IEEE 22nd International Conference on Emerging Technologies \& Factory Automation}, publisher = {IEEE}, address = {Piscataway}, isbn = {978-1-5090-6505-9}, doi = {https://doi.org/10.1109/ETFA.2017.8247771}, year = {2017}, language = {en} } @inproceedings{BanerjeeGrossmann2016, author = {Banerjee, Suprateek and Großmann, Daniel}, title = {An Electronic Device Description Language based approach for communication with dbms and file system in an industrial automation scenario}, booktitle = {ETFA'2016: 2016 IEEE 21st International Conference on Emerging Technologies and Factory Automation (ETFA)}, publisher = {IEEE}, address = {Piscataway}, isbn = {978-1-5090-1314-2}, doi = {https://doi.org/10.1109/ETFA.2016.7733682}, year = {2016}, language = {en} } @inproceedings{GrossmannBregullaBanerjeeetal.2014, author = {Großmann, Daniel and Bregulla, Markus and Banerjee, Suprateek and Schulz, Dirk and Braun, Roland}, title = {OPC UA server aggregation — The foundation for an internet of portals}, booktitle = {ETFA'2014: 19th IEEE International Conference on Emerging Technologies and Factory Automation}, publisher = {IEEE}, address = {Piscataway}, isbn = {978-1-4799-4846-8}, doi = {https://doi.org/10.1109/ETFA.2014.7005354}, year = {2014}, language = {en} } @inproceedings{MathiasGrossmannSequeira2019, author = {Mathias, Selvine George and Großmann, Daniel and Sequeira, Gerald Joy}, title = {A Comparison of Clustering Measures on Raw Signals of Welding Production Data}, booktitle = {Proceedings 2019 International Conference on Deep Learning and Machine Learning in Emerging Applications (Deep-ML)}, publisher = {IEEE}, address = {Piscataway}, isbn = {978-1-7281-2914-3}, doi = {https://doi.org/10.1109/Deep-ML.2019.00019}, pages = {55 -- 60}, year = {2019}, language = {en} } @inproceedings{SchmiedGrossmannDenk2019, author = {Schmied, Sebastian and Großmann, Daniel and Denk, Bastian}, title = {A systematic top-down information modelling approach for workshop-type manufacturing systems}, booktitle = {Proceedings 2019 24th IEEE International Conference on Emerging Technologies and Factory Automation (ETFA)}, publisher = {IEEE}, address = {Piscataway}, isbn = {978-1-7281-0303-7}, doi = {https://doi.org/10.1109/ETFA.2019.8869377}, pages = {1305 -- 1308}, year = {2019}, language = {en} } @inproceedings{MathiasSchmiedGrossmann2020, author = {Mathias, Selvine George and Schmied, Sebastian and Großmann, Daniel}, title = {Monitoring of discrete electrical signals from welding processes using data mining and IIoT approaches}, booktitle = {Proceedings, IEEE 32nd International Conference on Tools with Artificial Intelligence, (ICTAI 2020)}, publisher = {IEEE}, address = {Piscataway}, isbn = {978-1-7281-9228-4}, doi = {https://doi.org/10.1109/ICTAI50040.2020.00142}, pages = {911 -- 916}, year = {2020}, language = {en} } @inproceedings{SchmiedGrossmannMathiasetal.2020, author = {Schmied, Sebastian and Großmann, Daniel and Mathias, Selvine George and Mueller, Ralph}, title = {An approach for aggregation and historicization of production entities in the graph}, booktitle = {Proceedings 2020 25th IEEE International Conference on Emerging Technologies and Factory Automation (ETFA)}, publisher = {IEEE}, address = {Piscataway}, isbn = {978-1-7281-8956-7}, doi = {https://doi.org/10.1109/ETFA46521.2020.9211907}, pages = {1107 -- 1110}, year = {2020}, language = {en} } @inproceedings{MuellerGrossmannSchmiedetal.2020, author = {Mueller, Ralph and Großmann, Daniel and Schmied, Sebastian and Mathias, Selvine George}, title = {Validation of dynamic interoperability and virtual commissioning of production equipment in early development stages}, booktitle = {Proceedings 2020 25th IEEE International Conference on Emerging Technologies and Factory Automation (ETFA)}, publisher = {IEEE}, address = {Piscataway}, isbn = {978-1-7281-8956-7}, doi = {https://doi.org/10.1109/ETFA46521.2020.9211988}, pages = {1159 -- 1162}, year = {2020}, language = {en} } @inproceedings{MathiasSchmiedGrossmannetal.2020, author = {Mathias, Selvine George and Schmied, Sebastian and Großmann, Daniel and Mueller, Ralph and Mroß, Bj{\"o}rn}, title = {A Compliance Testing Structure for Implementation of Industry Standards through OPC UA}, booktitle = {Proceedings 2020 25th IEEE International Conference on Emerging Technologies and Factory Automation (ETFA)}, publisher = {IEEE}, address = {Piscataway}, isbn = {978-1-7281-8956-7}, doi = {https://doi.org/10.1109/ETFA46521.2020.9212119}, pages = {1091 -- 1094}, year = {2020}, language = {en} } @inproceedings{ThakurBeckMostaghimetal.2020, author = {Thakur, Akshay and Beck, Robert and Mostaghim, Sanaz and Großmann, Daniel}, title = {Machine Learning for evaluating Kaizens in Volkswagen Production System - An Industrial Case study}, booktitle = {Proceedings: 2020 IEEE 7th International Conference on Data Science and Advanced Analytics: DSAA 2020}, publisher = {IEEE}, address = {Piscataway}, isbn = {978-1-7281-8206-3}, doi = {https://doi.org/10.1109/DSAA49011.2020.00114}, pages = {781 -- 782}, year = {2020}, language = {en} } @inproceedings{SchmiedGrossmannMathiasetal.2020, author = {Schmied, Sebastian and Großmann, Daniel and Mathias, Selvine George and Banerjee, Suprateek}, title = {Vertical integration via dynamic aggregation of information in OPC UA}, booktitle = {Intelligent Information and Database Systems}, publisher = {Springer}, address = {Singapur}, isbn = {978-981-15-3379-2}, issn = {1865-0929}, doi = {https://doi.org/10.1007/978-981-15-3380-8_18}, pages = {204 -- 215}, year = {2020}, language = {en} }