@inproceedings{SchmidtkeSchintkeSchuett2018, author = {Schmidtke, Robert and Schintke, Florian and Sch{\"u}tt, Thorsten}, title = {From Application to Disk: Tracing I/O Through the Big Data Stack}, booktitle = {High Performance Computing ISC High Performance 2018 International Workshops, Frankfurt/Main, Germany, June 24 - 28, 2018, Revised Selected Papers, Workshop on Performance and Scalability of Storage Systems (WOPSSS)}, doi = {10.1007/978-3-030-02465-9_6}, pages = {89 -- 102}, year = {2018}, abstract = {Typical applications in data science consume, process and produce large amounts of data, making disk I/O one of the dominating — and thus worthwhile optimizing — factors of their overall performance. Distributed processing frameworks, such as Hadoop, Flink and Spark, hide a lot of complexity from the programmer when they parallelize these applications across a compute cluster. This exacerbates reasoning about I/O of both the application and the framework, through the distributed file system, such as HDFS, down to the local file systems. We present SFS (Statistics File System), a modular framework to trace each I/O request issued by the application and any JVM-based big data framework involved, mapping these requests to actual disk I/O. This allows detection of inefficient I/O patterns, both by the applications and the underlying frameworks, and builds the basis for improving I/O scheduling in the big data software stack.}, language = {en} } @inproceedings{SchmidtkeLaubenderSteinke2016, author = {Schmidtke, Robert and Laubender, Guido and Steinke, Thomas}, title = {Big Data Analytics on Cray XC Series DataWarp using Hadoop, Spark and Flink}, booktitle = {CUG Proceedings}, year = {2016}, language = {en} } @inproceedings{GholamiSchintke2021, author = {Gholami, Masoud and Schintke, Florian}, title = {Combining XOR and Partner Checkpointing for Resilient Multilevel Checkpoint/Restart}, booktitle = {2021 IEEE International Parallel and Distributed Processing Symposium (IPDPS)}, doi = {10.1109/IPDPS49936.2021.00036}, pages = {277 -- 288}, year = {2021}, language = {en} }