@misc{SchintkeReinefeldHaridietal., author = {Schintke, Florian and Reinefeld, Alexander and Haridi, Seif and Sch{\"u}tt, Thorsten}, title = {Enhanced Paxos Commit for Transactions on DHTs}, issn = {1438-0064}, url = {http://nbn-resolving.de/urn:nbn:de:0297-zib-11448}, number = {09-28}, abstract = {Key/value stores which are built on structured overlay networks often lack support for atomic transactions and strong data consistency among replicas. This is unfortunate, because consistency guarantees and transactions would allow a wide range of additional application domains to benefit from the inherent scalability and fault-tolerance of DHTs. The Scalaris key/value store supports strong data consistency and atomic transactions. It uses an enhanced Paxos Commit protocol with only four communication steps rather than six. This improvement was possible by exploiting information from the replica distribution in the DHT. Scalaris enables implementation of more reliable and scalable infrastructure for collaborative Web services that require strong consistency and atomic changes across multiple items.}, language = {en} } @misc{Schuett, type = {Master Thesis}, author = {Sch{\"u}tt, Thorsten}, title = {Synchronisation von verteilten Verzeichnisstrukturen}, url = {http://nbn-resolving.de/urn:nbn:de:0297-zib-10071}, school = {Zuse Institute Berlin (ZIB)}, abstract = {Das Management von großen Datenmengen spielt eine immer wichtigere Rolle, wie aktuelle Entwicklungen in der Hochenergiephysik [2] zeigen. F{\"u}r das DataGrid-Projekt zum Beispiel ist es notwendig, große Datenmengen auf mehrere Rechenzentren in Europa zu verteilen und die Daten untereinander zu synchronisieren. Auch innerhalb von Clustern gewinnen mit zunehmender Anzahl der Knoten Werkzeuge zur effizienten Synchronisation und Verteilung von Daten an Bedeutung. Im Rahmen dieser Arbeit wurde ein effizientes Verfahren zur Synchronisation von verteilten Verzeichnisstrukturen entwickelt und implementiert. Mit diesem Verfahren ist es m{\"o}glich, unabh{\"a}ngige {\"A}nderungen an beliebigen Repositories gleichzeitig durchzuf{\"u}hren. Das Verfahren ben{\"o}tigt keine zentrale Instanz, wodurch eine gegen{\"u}ber vielen existierenden Verfahren verbesserte Skalierbarkeit erreicht werden konnte. Dabei wurden Erkenntnisse aus der Graphentheorie eingesetzt und weiterentwickelt, um die Netzwerktopolo-gie und -handbreiten zwischen den Rechnern zu ber{\"u}cksichtigen. Durch die Verwendung einer Offline-Synchronisation werden {\"A}nderungen erst dann an andere Rechner propagiert, wenn der Nutzer dies anst{\"o}ßt. Das kann zum Beispiel nach einer abgeschlossenen Transaktion, die {\"A}nderungen an mehreren Dateien beinhaltet, angemessen sein.}, language = {de} } @misc{MoserHaridiShafaatetal., author = {Moser, Monika and Haridi, Seif and Shafaat, Tallat and Sch{\"u}tt, Thorsten and H{\"o}gqvist, Mikael and Reinefeld, Alexander}, title = {Transactional DHT Algorithms}, issn = {1438-0064}, url = {http://nbn-resolving.de/urn:nbn:de:0297-zib-11532}, number = {09-34}, abstract = {We present a framework for transactional data access on data stored in a DHT. It allows to atomically read and write items and to run distributed transactions consisting of a sequence of read and write operations on the items. Items are symmetrically replicated in order to achieve durability of data stored in the SON. To provide availability of items despite the unavailability of some replicas, operations on items are quorum-based. They make progress as long as a majority of replicas can be accessed. Our framework processes transactions optimistically with an atomic commit protocol that is based on Paxos atomic commit. We present algorithms for the whole framework with an event based notation. Additionally we discuss the problem of lookup inconsistencies and its implications on the one-copy serializability property of the transaction processing in our framework.}, language = {en} } @phdthesis{Schuett2010, author = {Sch{\"u}tt, Thorsten}, title = {Range queries in distributed hash tables}, year = {2010}, language = {en} } @inproceedings{DoebbelinSchuettReinefeld2012, author = {D{\"o}bbelin, Robert and Sch{\"u}tt, Thorsten and Reinefeld, Alexander}, title = {An Analysis of SMP Memory Allocators}, series = {Proceedings of the 41st International Conference on Parallel Processing Workshops (Fifth International Workshop on Parallel Programming Models and Systems Software for High-End Computing (P2S2))}, booktitle = {Proceedings of the 41st International Conference on Parallel Processing Workshops (Fifth International Workshop on Parallel Programming Models and Systems Software for High-End Computing (P2S2))}, publisher = {IEEE Computer Society}, doi = {10.1109/ICPPW.2012.10}, pages = {48 -- 54}, year = {2012}, language = {en} } @article{KruberHoegqvistSchuett2011, author = {Kruber, Nico and H{\"o}gqvist, Mikael and Sch{\"u}tt, Thorsten}, title = {The Benefits of Estimated Global Information in DHT Load Balancing}, series = {Cluster Computing and the Grid, IEEE International Symposium on}, volume = {0}, journal = {Cluster Computing and the Grid, IEEE International Symposium on}, publisher = {IEEE Computer Society}, address = {Los Alamitos, CA, USA}, doi = {10.1109/CCGrid.2011.11}, pages = {382 -- 391}, year = {2011}, language = {en} } @inproceedings{SchintkeReinefeldHaridietal.2010, author = {Schintke, Florian and Reinefeld, Alexander and Haridi, Seif and Sch{\"u}tt, Thorsten}, title = {Enhanced Paxos Commit for Transactions on DHTs}, series = {CCGRID}, booktitle = {CCGRID}, publisher = {IEEE}, doi = {10.1109/CCGRID.2010.41}, pages = {448 -- 454}, year = {2010}, language = {en} } @inproceedings{SchuettReinefeldSchintkeetal.2009, author = {Sch{\"u}tt, Thorsten and Reinefeld, Alexander and Schintke, Florian and Hennig, C.}, title = {Self-Adaptation in Large-Scale Systems: A Study on Structured Overlays Across Multiple Datacenters}, series = {Architectures and Languages for Self-Managing Distributed Systems (SelfMan@SASO)}, booktitle = {Architectures and Languages for Self-Managing Distributed Systems (SelfMan@SASO)}, doi = {10.1109/SASOW.2010.65}, year = {2009}, language = {en} } @inproceedings{SchuettMoserPlantikowetal.2008, author = {Sch{\"u}tt, Thorsten and Moser, Monika and Plantikow, Stefan and Schintke, Florian and Reinefeld, Alexander}, title = {A Transactional Scalable Distributed Data Store}, series = {1st IEEE International Scalable Computing Challenge, co-located with CCGrid'08}, booktitle = {1st IEEE International Scalable Computing Challenge, co-located with CCGrid'08}, year = {2008}, language = {en} } @inproceedings{SchuettReinefeldSchintkeetal.2009, author = {Sch{\"u}tt, Thorsten and Reinefeld, Alexander and Schintke, Florian and Hoffmann, Marie}, title = {Gossip-based Topology Inference for Efficient Overlay Mapping on Data Centers}, series = {Peer-to-Peer Computing}, booktitle = {Peer-to-Peer Computing}, doi = {10.1109/P2P.2009.5284553}, pages = {147 -- 150}, year = {2009}, language = {en} } @inproceedings{ReinefeldSchuett2009, author = {Reinefeld, Alexander and Sch{\"u}tt, Thorsten}, title = {Out-of-Core Parallel Frontier Search with MapReduce}, series = {HPCS}, volume = {5976}, booktitle = {HPCS}, publisher = {Springer}, doi = {10.1007/978-3-642-12659-8_24}, pages = {323 -- 336}, year = {2009}, language = {en} } @inproceedings{ReinefeldSchuettDoebbelin2010, author = {Reinefeld, Alexander and Sch{\"u}tt, Thorsten and D{\"o}bbelin, Robert}, title = {Very large pattern databases for heuristic search}, series = {HPDC}, booktitle = {HPDC}, editor = {Hariri, Salim and Keahey, Kate}, publisher = {ACM}, doi = {10.1145/1851476.1851592}, pages = {803 -- 809}, year = {2010}, language = {en} } @incollection{ReinefeldSchintkeSchuettetal.2009, author = {Reinefeld, Alexander and Schintke, Florian and Sch{\"u}tt, Thorsten and Haridi, Seif}, title = {A Scalable, Transactional Data Store for Future Internet Services}, series = {Towards the Future Internet - A European Research Perspective}, booktitle = {Towards the Future Internet - A European Research Perspective}, editor = {Tselentis, G. and Domingue, J. and Galis, A. and Gavras, A. and Hausheer, D. and Krco, S. and Lotz, V. and Zahariadis, T.}, publisher = {IOS Press}, doi = {10.3233/978-1-60750-007-0-148}, pages = {148 -- 159}, year = {2009}, language = {en} } @inproceedings{CostaPierreReinefeldetal.2008, author = {Costa, Paolo and Pierre, Guillaume and Reinefeld, Alexander and Sch{\"u}tt, Thorsten and van Steen, Maarten}, title = {Sloppy Management of Structured P2P Services}, series = {Proceedings of the 3\$^rd\$ International Workshop on Hot Topics in Autonomic Computing (HotAC III), co-located with IEEE ICAC'08}, booktitle = {Proceedings of the 3\$^rd\$ International Workshop on Hot Topics in Autonomic Computing (HotAC III), co-located with IEEE ICAC'08}, address = {Chicago, IL, USA}, year = {2008}, language = {en} } @inproceedings{HoegqvistHaridiKruberetal.2008, author = {H{\"o}gqvist, Mikael and Haridi, Seif and Kruber, Nico and Reinefeld, Alexander and Sch{\"u}tt, Thorsten}, title = {Using Global Information for Load Balancing in DHTs}, series = {Workshop on Decentralized Self Management for Grids, P2P, and User Communities}, booktitle = {Workshop on Decentralized Self Management for Grids, P2P, and User Communities}, year = {2008}, language = {en} } @misc{AndrzejakReinefeldSchintkeetal.2008, author = {Andrzejak, Artur and Reinefeld, Alexander and Schintke, Florian and Sch{\"u}tt, Thorsten and Mastroianni, Carlo and Fragopoulou, Paraskevi and Kondo, Derrick and Malecot, Paul and Cosmin Silaghi, Gheorghe and Moura Silva, Luis and Trunfio, Paolo and Zeinalipour-Yazti, Demetris and Zimeo, Eugenio}, title = {Grid Architectural Issues: State-of-the-art and Future Trends}, number = {WHP-0004}, publisher = {Institute on Architectural Issues: Scalability, Dependability, Adaptability, CoreGRID - Network of Excellence}, year = {2008}, language = {en} } @inproceedings{ShafaatMoserGhodsietal.2008, author = {Shafaat, Tallat and Moser, Monika and Ghodsi, Ali and Sch{\"u}tt, Thorsten and Haridi, Seif and Reinefeld, Alexander}, title = {Key-Based Consistency and Availability in Structured Overlay Networks}, series = {International ICST Conference on Scalable Information Systems}, booktitle = {International ICST Conference on Scalable Information Systems}, year = {2008}, language = {en} } @inproceedings{ShafaatMoserGhodsietal.2008, author = {Shafaat, Tallat and Moser, Monika and Ghodsi, Ali and Sch{\"u}tt, Thorsten and Reinefeld, Alexander}, title = {On Consistency of Data in Structured Overlay Networks}, series = {Coregrid Integration Workshop}, booktitle = {Coregrid Integration Workshop}, year = {2008}, language = {en} } @article{SchuettSchintkeReinefeld2008, author = {Sch{\"u}tt, Thorsten and Schintke, Florian and Reinefeld, Alexander}, title = {Range Queries on structured overlay networks}, series = {Computer Communications}, volume = {31}, journal = {Computer Communications}, number = {2}, publisher = {Elsevier}, doi = {10.1016/j.comcom.2007.08.027}, pages = {280 -- 291}, year = {2008}, language = {en} } @inproceedings{SchuettSchintkeReinefeld2008, author = {Sch{\"u}tt, Thorsten and Schintke, Florian and Reinefeld, Alexander}, title = {Scalaris: Reliable Transactional P2P Key/Value}, series = {ERLANG '08: Proceedings of the 7th ACM SIGPLAN workshop on ERLANG}, booktitle = {ERLANG '08: Proceedings of the 7th ACM SIGPLAN workshop on ERLANG}, doi = {10.1145/1411273.1411280}, pages = {41 -- 47}, year = {2008}, language = {en} } @inproceedings{SchuettDoebbelinReinefeld2013, author = {Sch{\"u}tt, Thorsten and D{\"o}bbelin, Robert and Reinefeld, Alexander}, title = {Forward Perimeter Search with Controlled Use of Memory}, series = {International Joint Conference on Artificial Intelligence, IJCAI-13, Beijing}, booktitle = {International Joint Conference on Artificial Intelligence, IJCAI-13, Beijing}, year = {2013}, language = {en} } @article{SchuettReinefeldDoebbelin, author = {Sch{\"u}tt, Thorsten and Reinefeld, Alexander and D{\"o}bbelin, Robert}, title = {MR-search: massively parallel heuristic search}, series = {Concurrency and Computation: Practice and Experience}, volume = {25}, journal = {Concurrency and Computation: Practice and Experience}, number = {1}, doi = {10.1002/cpe.1833}, pages = {40 -- 54}, language = {en} } @article{ReinefeldSchuettDoebbelin, author = {Reinefeld, Alexander and Sch{\"u}tt, Thorsten and D{\"o}bbelin, Robert}, title = {Analyzing the performance of SMP memory allocators with iterative MapReduce applications}, series = {Parallel Computing}, volume = {39}, journal = {Parallel Computing}, number = {12}, publisher = {Elsevier}, doi = {10.1016/j.parco.2013.09.007}, pages = {879 -- 889}, language = {en} } @inproceedings{KleineweberReinefeldSchuett, author = {Kleineweber, Christoph and Reinefeld, Alexander and Sch{\"u}tt, Thorsten}, title = {QoS-aware Storage Virtualization for Cloud File Systems}, series = {Proceedings of the 1st ACM International Workshop on Programmable File Systems}, booktitle = {Proceedings of the 1st ACM International Workshop on Programmable File Systems}, doi = {10.1145/2603941.2603944}, pages = {19 -- 26}, language = {en} } @inproceedings{WeinholdLackorzynskiBierbaumetal., author = {Weinhold, Carsten and Lackorzynski, Adam and Bierbaum, Jan and K{\"u}ttler, Martin and Planeta, Maksym and H{\"a}rtig, Hermann and Shiloh, Amnon and Levy, Ely and Ben-Nun, Tal and Barak, Amnon and Steinke, Thomas and Sch{\"u}tt, Thorsten and Fajerski, Jan and Reinefeld, Alexander and Lieber, Matthias and Nagel, Wolfgang}, title = {FFMK: A Fast and Fault-tolerant Microkernel-based System for Exascale Computing}, series = {SPPEXA Symposium 2016}, booktitle = {SPPEXA Symposium 2016}, doi = {10.1007/978-3-319-40528-5_18}, language = {en} } @inproceedings{FajerskiNoackReinefeldetal., author = {Fajerski, J. and Noack, Matthias and Reinefeld, Alexander and Schintke, Florian and Sch{\"u}tt, Thorsten and Steinke, Thomas}, title = {Fast In-Memory Checkpointing with POSIX API for Legacy Exascale-Applications}, series = {SPPEXA Symposium 2016}, booktitle = {SPPEXA Symposium 2016}, doi = {10.1007/978-3-319-40528-5_19}, language = {en} } @incollection{CoutinhoStillwellArgyrakietal., author = {Coutinho, Jose and Stillwell, Mark and Argyraki, Katerina and Ioannidis, George and Iordache, Anca and Kleineweber, Christoph and Koliousis, Alexandros and McGlone, John and Pierre, Guillaume and Ragusa, Carmelo and Sanders, Peter and Sch{\"u}tt, Thorsten and Yu, Teng and Wolf, Alexander}, title = {The HARNESS Platform: A Hardware- and Network-Enhanced Software System for Cloud Computing}, series = {Software Architecture for Big Data and the Cloud}, booktitle = {Software Architecture for Big Data and the Cloud}, publisher = {Morgan Kaufmann}, isbn = {9780128054673}, pages = {323 -- 351}, language = {en} } @inproceedings{DoebbelinSchuettReinefeld2013, author = {D{\"o}bbelin, Robert and Sch{\"u}tt, Thorsten and Reinefeld, Alexander}, title = {Building Large Compressed PDBs for the Sliding Tile Puzzle}, series = {Workshop on Computer Games}, booktitle = {Workshop on Computer Games}, year = {2013}, language = {en} } @article{AllenDavisGoodaleetal.2005, author = {Allen, Gabrielle and Davis, K. and Goodale, Tom and Hutanu, Andrei and Kaiser, Hartmut and Kielmann, Thilo and Merzky, Andre and Nieuwpoort, R. and Reinefeld, Alexander and Schintke, Florian and Sch{\"u}tt, Thorsten and Seidel, Edward and Ullmer, Brygg}, title = {The Grid Application Toolkit: Toward Generic and Easy Appliction Programming Interfaces for the Grid}, series = {Procceedings of the IEEE}, volume = {93}, journal = {Procceedings of the IEEE}, number = {3}, publisher = {IEEE Computer Society Press}, doi = {10.1109/JPROC.2004.842755}, pages = {534 -- 550}, year = {2005}, language = {en} } @inproceedings{SchuettMerzkyHutanuetal.2004, author = {Sch{\"u}tt, Thorsten and Merzky, Andr{\´e} and Hutanu, Andrei and Schintke, Florian}, title = {Remote partial file access using compact pattern descriptions}, series = {4th IEEE/ACM International Symposium on Cluster Computing and the Grid (CCGrid 2004), April 19-22, 2004, Chicago, Illinois, USA}, booktitle = {4th IEEE/ACM International Symposium on Cluster Computing and the Grid (CCGrid 2004), April 19-22, 2004, Chicago, Illinois, USA}, publisher = {IEEE Computer Society}, doi = {10.1109/CCGrid.2004.1336604}, pages = {482 -- 489}, year = {2004}, language = {en} } @incollection{ReinefeldSchintkeSchuett2004, author = {Reinefeld, Alexander and Schintke, Florian and Sch{\"u}tt, Thorsten}, title = {Scalable and Self-Optimizing Data Grids}, series = {Annual Review of Scalable Computing}, volume = {6}, booktitle = {Annual Review of Scalable Computing}, number = {chapter 2}, publisher = {World Scientific}, doi = {10.1142/9789812794727_0002}, pages = {30 -- 60}, year = {2004}, language = {en} } @misc{ReinefeldSchintkeSchuett2007, author = {Reinefeld, Alexander and Schintke, Florian and Sch{\"u}tt, Thorsten}, title = {Vorrichtung und Verfahren zum Abrufen / Speichern von elektronischen Daten in einem System mit mehreren Datenverarbeitungseinheiten}, publisher = {Europ{\"a}isches Patent EP 1 744 257 A1}, year = {2007}, language = {en} } @misc{ReinefeldSchintkeSchuett2007, author = {Reinefeld, Alexander and Schintke, Florian and Sch{\"u}tt, Thorsten}, title = {Vorrichtung und Verfahren zum Speichern / Abrufen von Objekten mit mehrdimensional adressierten, elektronischen Daten}, publisher = {Europ{\"a}ische Patentanmeldung Nr. 06012030.0 vom 12.06.2006, Patent Nr: 1868114}, year = {2007}, language = {en} } @inproceedings{SchintkeSchuettReinefeld2003, author = {Schintke, Florian and Sch{\"u}tt, Thorsten and Reinefeld, Alexander}, title = {A Framework for Self-Optimizing Grids Using P2P Components}, series = {14th International Workshop on Database and Expert Systems Applications (DEXA'03), September 1-5, 2003, Prague, Czech Republic}, booktitle = {14th International Workshop on Database and Expert Systems Applications (DEXA'03), September 1-5, 2003, Prague, Czech Republic}, publisher = {IEEE Computer Society}, doi = {10.1109/DEXA.2003.1232101}, pages = {689 -- 693}, year = {2003}, language = {en} } @inproceedings{SchuettSchintkeReinefeld2003, author = {Sch{\"u}tt, Thorsten and Schintke, Florian and Reinefeld, Alexander}, title = {Efficient Synchronization of Replicated Data in Distributed Systems}, series = {Computational Science - ICCS 2003, International Conference, Melbourne, Australia and St. Petersburg, Russia, June 2-4, 2003. Proceedings, Part I}, volume = {2657}, booktitle = {Computational Science - ICCS 2003, International Conference, Melbourne, Australia and St. Petersburg, Russia, June 2-4, 2003. Proceedings, Part I}, editor = {Sloot, Peter and Abramson, David and Bogdanov, Alexander and Dongarra, Jack and Zomaya, Albert and Gorbachev, Yuri}, publisher = {Springer}, doi = {10.1007/3-540-44860-8_28}, pages = {274 -- 283}, year = {2003}, language = {en} } @inproceedings{AndrzejakReinefeldSchintkeetal.2006, author = {Andrzejak, Artur and Reinefeld, Alexander and Schintke, Florian and Sch{\"u}tt, Thorsten}, title = {On Adaptability in Grid Systems}, series = {Future Generation Grids: Proceedings of the Workshop on Future Generation Grids, November 1-5, 2004, Dagstuhl, Germany}, booktitle = {Future Generation Grids: Proceedings of the Workshop on Future Generation Grids, November 1-5, 2004, Dagstuhl, Germany}, editor = {Getov, Vladimir and Laforenza, Domenico and Reinefeld, Alexander}, publisher = {Springer}, doi = {10.1007/978-0-387-29445-2_2}, pages = {29 -- 46}, year = {2006}, language = {en} } @inproceedings{SchuettSchintkeReinefeld2006, author = {Sch{\"u}tt, Thorsten and Schintke, Florian and Reinefeld, Alexander}, title = {Structured Overlay without Consistent Hashing: Empirical Results}, series = {Sixth Workshop on Global and Peer-to-Peer Computing (GP2PC'06) at Sixth IEEE International Symposium on Cluster Computing and the Grid (CCGrid 2006), 16-19 May 2006, Singapore}, booktitle = {Sixth Workshop on Global and Peer-to-Peer Computing (GP2PC'06) at Sixth IEEE International Symposium on Cluster Computing and the Grid (CCGrid 2006), 16-19 May 2006, Singapore}, publisher = {IEEE Computer Society}, doi = {10.1109/CCGRID.2006.175}, pages = {8}, year = {2006}, language = {en} } @inproceedings{SchuettSchintkeReinefeld2007, author = {Sch{\"u}tt, Thorsten and Schintke, Florian and Reinefeld, Alexander}, title = {A Structured Overlay for Multi-dimensional Range Queries}, series = {Euro-Par Conference}, volume = {4641}, booktitle = {Euro-Par Conference}, editor = {Kermarrec, Anne-Marie}, publisher = {Springer}, doi = {10.1007/978-3-540-74466-5_54}, pages = {503 -- 513}, year = {2007}, language = {en} } @misc{ReinefeldSchintkeSchuett2007, author = {Reinefeld, Alexander and Schintke, Florian and Sch{\"u}tt, Thorsten}, title = {Device and method for retrieving / storing electronic data in a system with a plurality of data processing units}, publisher = {United States Patent Application Publication No. US 2007/0165619 A1}, year = {2007}, language = {en} } @inproceedings{AnderssonSachsTumaetal., author = {Andersson, Stefan and Sachs, Stephen and Tuma, Christian and Sch{\"u}tt, Thorsten}, title = {DataWarp: First Experiences}, series = {Cray User Group}, booktitle = {Cray User Group}, language = {en} } @inproceedings{SkrzypczakSchintkeSchuett, author = {Skrzypczak, Jan and Schintke, Florian and Sch{\"u}tt, Thorsten}, title = {Linearizable State Machine Replication of State-Based CRDTs without Logs}, series = {Proceedings of the 2019 ACM Symposium on Principles of Distributed Computing, PODC 2019}, booktitle = {Proceedings of the 2019 ACM Symposium on Principles of Distributed Computing, PODC 2019}, doi = {10.1145/3293611.3331568}, pages = {455 -- 457}, abstract = {General solutions of state machine replication have to ensure that all replicas apply the same commands in the same order, even in the presence of failures. Such strict ordering incurs high synchronization costs due to the use of distributed consensus or a leader. This paper presents a protocol for linearizable state machine replication of conflict-free replicated data types (CRDTs) that neither requires consensus nor a leader. By leveraging the properties of state-based CRDTs—in particular the monotonic growth of a join semilattice—synchronization overhead is greatly reduced. In addition, updates just need a single round trip and modify the state 'in-place' without the need for a log. Furthermore, the message size overhead for coordination consists of a single counter per message. While reads in the presence of concurrent updates are not wait-free without a coordinator, we show that more than 97 \% of reads can be handled in one or two round trips under highly concurrent accesses. Our protocol achieves high throughput without auxiliary processes such as command log management or leader election. It is well suited for all practical scenarios that need linearizable access on CRDT data on a fine-granular scale.}, language = {en} } @article{SkrzypczakSchintkeSchuett, author = {Skrzypczak, Jan and Schintke, Florian and Sch{\"u}tt, Thorsten}, title = {RMWPaxos: Fault-Tolerant In-Place Consensus Sequences}, series = {IEEE Transactions on Parallel and Distributed Systems}, volume = {31}, journal = {IEEE Transactions on Parallel and Distributed Systems}, number = {10}, issn = {1045-9219}, doi = {10.1109/TPDS.2020.2981891}, pages = {2392 -- 2405}, language = {en} } @inproceedings{HartungSchintkeSchuett, author = {Hartung, Marc and Schintke, Florian and Sch{\"u}tt, Thorsten}, title = {Pinpoint Data Races via Testing and Classification}, series = {2019 IEEE International Symposium on Software Reliability Engineering Workshops (ISSREW); 3rd International Workshop on Software Faults (IWSF 2019)}, booktitle = {2019 IEEE International Symposium on Software Reliability Engineering Workshops (ISSREW); 3rd International Workshop on Software Faults (IWSF 2019)}, doi = {10.1109/ISSREW.2019.00100}, pages = {386 -- 393}, language = {en} } @inproceedings{SchmidtkeSchintkeSchuett, author = {Schmidtke, Robert and Schintke, Florian and Sch{\"u}tt, Thorsten}, title = {From Application to Disk: Tracing I/O Through the Big Data Stack}, series = {High Performance Computing ISC High Performance 2018 International Workshops, Frankfurt/Main, Germany, June 24 - 28, 2018, Revised Selected Papers, Workshop on Performance and Scalability of Storage Systems (WOPSSS)}, booktitle = {High Performance Computing ISC High Performance 2018 International Workshops, Frankfurt/Main, Germany, June 24 - 28, 2018, Revised Selected Papers, Workshop on Performance and Scalability of Storage Systems (WOPSSS)}, doi = {10.1007/978-3-030-02465-9_6}, pages = {89 -- 102}, abstract = {Typical applications in data science consume, process and produce large amounts of data, making disk I/O one of the dominating — and thus worthwhile optimizing — factors of their overall performance. Distributed processing frameworks, such as Hadoop, Flink and Spark, hide a lot of complexity from the programmer when they parallelize these applications across a compute cluster. This exacerbates reasoning about I/O of both the application and the framework, through the distributed file system, such as HDFS, down to the local file systems. We present SFS (Statistics File System), a modular framework to trace each I/O request issued by the application and any JVM-based big data framework involved, mapping these requests to actual disk I/O. This allows detection of inefficient I/O patterns, both by the applications and the underlying frameworks, and builds the basis for improving I/O scheduling in the big data software stack.}, language = {en} } @article{SkrzypczakSchintkeSchuett, author = {Skrzypczak, Jan and Schintke, Florian and Sch{\"u}tt, Thorsten}, title = {Linearizable State Machine Replication of State-Based CRDTs without Logs}, series = {arXiv}, journal = {arXiv}, abstract = {General solutions of state machine replication have to ensure that all replicas apply the same commands in the same order, even in the presence of failures. Such strict ordering incurs high synchronization costs caused by distributed consensus or by the use of a leader. This paper presents a protocol for linearizable state machine replication of conflict-free replicated data types (CRDTs) that neither requires consensus nor a leader. By leveraging the properties of state-based CRDTs - in particular the monotonic growth of a join semilattice - synchronization overhead is greatly reduced. In addition, updates just need a single round trip and modify the state `in-place' without the need for a log. Furthermore, the message size overhead for coordination consists of a single counter per message. While reads in the presence of concurrent updates are not wait-free without a coordinator, we show that more than 97\% of reads can be handled in one or two round trips under highly concurrent accesses. Our protocol achieves high throughput without auxiliary processes like command log management or leader election. It is well suited for all practical scenarios that need linearizable access on CRDT data on a fine-granular scale.}, language = {en} } @article{SchuettSchintkeSkrzypczak, author = {Sch{\"u}tt, Thorsten and Schintke, Florian and Skrzypczak, Jan}, title = {Transactions on Red-black and AVL trees in NVRAM}, series = {arXiv}, journal = {arXiv}, language = {en} } @misc{ReinefeldSchintkeSchuett, author = {Reinefeld, Alexander and Schintke, Florian and Sch{\"u}tt, Thorsten}, title = {P2P Routing of Range Queries in Skewed Multidimensional Data Sets}, issn = {1438-0064}, url = {http://nbn-resolving.de/urn:nbn:de:0297-zib-9671}, number = {07-23}, abstract = {We present a middleware to store multidimensional data sets on Internet-scale distributed systems and to efficiently perform range queries on them. Our structured overlay network \emph{SONAR (Structured Overlay Network with Arbitrary Range queries)} puts keys which are adjacent in the key space on logically adjacent nodes in the overlay and is thereby able to process multidimensional range queries with a single logarithmic data lookup and local forwarding. The specified ranges may have arbitrary shapes like rectangles, circles, spheres or polygons. Empirical results demonstrate the routing performance of SONAR on several data sets, ranging from real-world data to artificially constructed worst case distributions. We study the quality of SONAR's routing information which is based on local knowledge only and measure the indegree of the overlay nodes to find potential hot spots in the routing process. We show that SONAR's routing table is self-adjusting, even under extreme situations, keeping always a maximum of \$\lceil \log N \rceil\$ routing entries.}, language = {en} } @misc{SchuettSchintkeReinefeld, author = {Sch{\"u}tt, Thorsten and Schintke, Florian and Reinefeld, Alexander}, title = {Chord\#: Structured Overlay Network for Non-Uniform Load-Distribution}, url = {http://nbn-resolving.de/urn:nbn:de:0297-zib-8736}, number = {05-40}, abstract = {\newcommand{\chordsharp}{Chord\$^\\#\#\$} Data lookup is a fundamental problem in peer-to-peer systems: Given a key, find the node that stores the associated object. Chord and other P2P algorithms use distributed hash tables (DHTs) to distribute the keys and nodes evenly across a logical ring. Using an efficient routing strategy, DHTs provide a routing performance of \$O (\log N)\$ in networks of \$N\$ nodes. While the routing performance has been shown to be optimal, the uniform key distribution makes it impossible for DHTs to support range queries. For range queries, consecutive keys must be stored on lo gically neighboring nodes. In this paper, we present an enhancement of Chord that eliminates the hash function while keeping the same routing performance. The resulting algorithm, named \chordsharp{}, provides a richer function ality while maintaining the same complexity. In addition to Chord, \chordsharp{} adapts to load imbalance.}, language = {en} } @misc{DoebbelinSchuettReinefeld, author = {D{\"o}bbelin, Robert and Sch{\"u}tt, Thorsten and Reinefeld, Alexander}, title = {Building Large Compressed PDBs for the Sliding Tile Puzzle}, issn = {1438-0064}, url = {http://nbn-resolving.de/urn:nbn:de:0297-zib-18095}, abstract = {The performance of heuristic search algorithms depends crucially on the effectiveness of the heuristic. A pattern database (PDB) is a powerful heuristic in the form of a pre-computed lookup table. Larger PDBs provide better bounds and thus allow more cut-offs in the search process. Today, the largest PDB for the 24-puzzle is a 6-6-6-6 PDB with a size of 486 MB. We created 8-8-8, 9-8-7 and 9-9-6 PDBs that are three orders of magnitude larger (up to 1.4 TB) than the 6-6-6-6 PDB. We show how to compute such large PDBs and we present statistical and empirical data on their efficiency. The largest single PDB gives on average an 8-fold improvement over the 6-6-6-6 PDB. Combining several large PDBs gives on average an 12-fold improvement.}, language = {en} } @inproceedings{WendeNoackSchuettetal., author = {Wende, Florian and Noack, Matthias and Sch{\"u}tt, Thorsten and Sachs, Stephen and Steinke, Thomas}, title = {Application Performance on a Cray XC30 Evaluation System with Xeon Phi Coprocessors at HLRN-III}, series = {Cray User Group}, booktitle = {Cray User Group}, language = {en} } @inproceedings{SalemSchuettSchintkeetal., author = {Salem, Farouk and Sch{\"u}tt, Thorsten and Schintke, Florian and Reinefeld, Alexander}, title = {Scheduling Data Streams for Low Latency and High Throughput on a Cray XC40 Using Libfabric}, series = {CUG Conference Proceedings}, booktitle = {CUG Conference Proceedings}, abstract = {Achieving efficient many-to-many communication on a given network topology is a challenging task when many data streams from different sources have to be scattered concurrently to many destinations with low variance in arrival times. In such scenarios, it is critical to saturate but not to congest the bisectional bandwidth of the network topology in order to achieve a good aggregate throughput. When there are many concurrent point-to-point connections, the communication pattern needs to be dynamically scheduled in a fine-grained manner to avoid network congestion (links, switches), overload in the node's incoming links, and receive buffer overflow. Motivated by the use case of the Compressed Baryonic Matter experiment (CBM), we study the performance and variance of such communication patterns on a Cray XC40 with different routing schemes and scheduling approaches. We present a distributed Data Flow Scheduler (DFS) that reduces the variance of arrival times from all sources at least 30 times and increases the achieved aggregate bandwidth by up to 50\%.}, language = {en} } @article{SalemSchintkeSchuettetal., author = {Salem, Farouk and Schintke, Florian and Sch{\"u}tt, Thorsten and Reinefeld, Alexander}, title = {Scheduling data streams for low latency and high throughput on a Cray XC40 using Libfabric}, series = {Concurrency and Computation Practice and Experience}, volume = {32}, journal = {Concurrency and Computation Practice and Experience}, number = {20}, doi = {10.1002/cpe.5563}, pages = {1 -- 14}, language = {en} } @article{SalemSchintkeSchuettetal., author = {Salem, Farouk and Schintke, Florian and Sch{\"u}tt, Thorsten and Reinefeld, Alexander}, title = {Improving the throughput of a scalable FLESnet using the Data-Flow Scheduler}, series = {CBM Progress Report 2018}, journal = {CBM Progress Report 2018}, isbn = {978-3-9815227-6-1}, doi = {10.15120/GSI-2019-01018}, pages = {149 -- 150}, language = {en} } @article{SalemSchintkeSchuettetal., author = {Salem, Farouk and Schintke, Florian and Sch{\"u}tt, Thorsten and Reinefeld, Alexander}, title = {Supporting various interconnects in FLESnet using Libfabric}, series = {CBM Progress Report 2016}, journal = {CBM Progress Report 2016}, isbn = {978-3-9815227-4-7}, pages = {159 -- 160}, language = {en} } @article{SalemSchintkeSchuettetal., author = {Salem, Farouk and Schintke, Florian and Sch{\"u}tt, Thorsten and Reinefeld, Alexander}, title = {Data-flow scheduling for a scalable FLESnet}, series = {CBM Progress Report 2017}, journal = {CBM Progress Report 2017}, isbn = {978-3-9815227-5-4}, doi = {10.15120/GSI-2018-00485}, pages = {130 -- 131}, language = {en} } @inproceedings{GholamiSchintkeSchuett, author = {Gholami, Masoud and Schintke, Florian and Sch{\"u}tt, Thorsten}, title = {Checkpoint Scheduling for Shared Usage of Burst-Buffers in Supercomputers}, series = {Proceedings of the 47th International Conference on Parallel Processing Companion; SRMPDS 2018: The 14th International Workshop on Scheduling and Resource Management for Parallel and Distributed Systems}, booktitle = {Proceedings of the 47th International Conference on Parallel Processing Companion; SRMPDS 2018: The 14th International Workshop on Scheduling and Resource Management for Parallel and Distributed Systems}, doi = {10.1145/3229710.3229755}, pages = {44:1 -- 44:10}, abstract = {User-defined and system-level checkpointing have contrary properties. While user-defined checkpoints are smaller and simpler to recover, system-level checkpointing better knows the global system's state and parameters like the expected mean time to failure (MTTF) per node. Both approaches lead to non-optimal checkpoint time, intervals, sizes, or I/O bandwidth when concurrent checkpoints conflict and compete for it. We combine user-defined and system-level checkpointing to exploit the benefits and avoid the drawbacks of each other. Thus, applications frequently offer to create checkpoints. The system accepts such offers according to the current status and implied costs to recalculate from the last checkpoint or denies them, i.e., immediately lets continue the application without checkpoint creation. To support this approach, we develop economic models for multi-application checkpointing on shared I/O resources that are dedicated for checkpointing (e.g. burst-buffers) by defining an appropriate goal function and solving a global optimization problem. Using our models, the checkpoints of applications on a supercomputer are scheduled to effectively use the available I/O bandwidth and minimize the failure overhead (checkpoint creations plus recalculations). Our simulations show an overall reduction in failure overhead of all nodes of up to 30\% for a typical supercomputer workload (HLRN). We can also derive the most cost effective burst-buffer bandwidth for a given node's MTTF and application workload.}, language = {en} } @inproceedings{WeinholdLackorzynskiBierbaumetal., author = {Weinhold, Carsten and Lackorzynski, Adam and Bierbaum, Jan and K{\"u}ttler, Martin and Planeta, Maksym and Weisbach, Hannes and Hille, Matthias and H{\"a}rtig, Hermann and Margolin, Alexander and Sharf, Dror and Levy, Ely and Gak, Pavel and Barak, Amnon and Gholami, Masoud and Schintke, Florian and Sch{\"u}tt, Thorsten and Reinefeld, Alexander and Lieber, Matthias and Nagel, Wolfgang}, title = {FFMK: A Fast and Fault-Tolerant Microkernel-Based System for Exascale Computing}, series = {Software for Exascale Computing - SPPEXA 2016-2019}, booktitle = {Software for Exascale Computing - SPPEXA 2016-2019}, publisher = {Springer}, doi = {10.1007/978-3-030-47956-5_16}, pages = {483 -- 516}, language = {en} } @misc{GholamiSchintkeSchuettetal., author = {Gholami, Masoud and Schintke, Florian and Sch{\"u}tt, Thorsten and Reinefeld, Alexander}, title = {Modeling Checkpoint Schedules for Concurrent HPC Applications}, series = {CoSaS 2018 International Symposium on Computational Science at Scale}, journal = {CoSaS 2018 International Symposium on Computational Science at Scale}, language = {en} }