@article{MauererJoblinTamburrietal., author = {Mauerer, Wolfgang and Joblin, Mitchell and Tamburri, Damian and Paradis, Carlos and Kazman, Rick and Apel, Sven}, title = {In Search of Socio-Technical Congruence: A Large-Scale Longitudinal Study}, series = {IEEE Transactions on Software Engineering (TSE)}, volume = {48}, journal = {IEEE Transactions on Software Engineering (TSE)}, number = {8}, publisher = {IEEE}, doi = {10.1109/TSE.2021.3082074}, pages = {3159 -- 3184}, abstract = {This paper describes a large-scale empirical study investigating the relevance of socio-technical congruence over key basic software quality metrics, namely, bugs and churn. That is, we explore whether alignment or misalignment of social communication structures and technical dependencies in large software projects influences software quality. To this end, we have defined a quantitative and operational notion of socio-technical congruence, which we call /socio-technical motif congruence/ (STMC). STMC is a measure of the degree to which developers working on the same file or on two related files, need to communicate. As socio-technical congruence is a complex and multi-faceted phenomenon, the interpretability of the results is one of our main concerns, so we have employed a careful mixed-methods statistical analysis. In particular, we provide analyses with similar techniques as employed by seminal work in the field to ensure comparability of our results with the existing body of work. The major result of our study, based on an analysis of 25 large open-source projects, is that STMC is /not/ related to project quality measures---software bugs and churn---in any temporal scenario. That is, we find no statistical relationship between the alignment of developer tasks and developer communications on one hand, and project outcomes on the other hand. We conclude that, wherefore congruence does matter as literature shows, then its measurable effect lies elsewhere.}, language = {en} } @inproceedings{HoessParadisKazmanetal., author = {Hoess, Nicole and Paradis, Carlos and Kazman, Rick and Mauerer, Wolfgang}, title = {Does the Tool Matter? Exploring Some Causes of Threats to Validity in Mining Software Repositories}, series = {2025 IEEE International Conference on Software Analysis, Evolution and Reengineering (SANER), Montreal, QC, Canada, March, 4-7, 2025}, booktitle = {2025 IEEE International Conference on Software Analysis, Evolution and Reengineering (SANER), Montreal, QC, Canada, March, 4-7, 2025}, publisher = {IEEE}, isbn = {979-8-3315-3510-0}, doi = {10.1109/SANER64311.2025.00067}, pages = {645 -- 656}, abstract = {Software repositories are an essential source of information for software engineering research on topics such as project evolution and developer collaboration. Appropriate mining tools and analysis pipelines are therefore an indispensable precondition for many research activities. Ideally, valid results should not depend on technical details of data collection and processing. It is, however, widely acknowledged that mining pipelines are complex, with a multitude of implementation decisions made by tool authors based on their interests and assumptions. This raises the questions if (and to what extent) tools agree on their results and are interchangeable. In this study, we use two tools to extract and analyse ten large software projects, quantitatively and qualitatively comparing results and derived data to better understand this concern. We analyse discrepancies from a technical point of view, and adjust code and parametrisation to minimise replication differences. Our results indicate that despite similar trends, even simple metrics such as the numbers of commits and developers may differ by up to 500\%. We find that such substantial differences are often caused by minor technical details. We show how tool-level and data post-processing changes can overcome these issues, but find they may require considerable efforts. We summarise identified causes in our lessons learned to help researchers and practitioners avoid common pitfalls, and reflect on implementation decisions and their influence in ensuring obtained data meets explicit and implicit expectations. Our findings lead us to hypothesise that similar uncertainties exist in other analysis tools, which may limit the validity of conclusions drawn in tool-centric research.}, language = {en} } @misc{MauererJoblinTamburrietal., author = {Mauerer, Wolfgang and Joblin, Mitchell and Tamburri, Damian Andrew and Paradis, Carlos and Kazman, Rick and Apel, Sven}, title = {In Search of Socio-Technical Congruence: A Large-Scale Longitudinal Study [Data set]}, doi = {10.5281/zenodo.4766388}, language = {en} } @unpublished{HoessParadisKazmanetal., author = {H{\"o}ß, Nicole and Paradis, Carlos and Kazman, Rick and Mauerer, Wolfgang}, title = {Oops!... I did it again. Conclusion (In-)Stability in Quantitative Empirical Software Engineering: A Large-Scale Analysis}, doi = {10.48550/arXiv.2510.06844}, pages = {80}, abstract = {Context: Mining software repositories is a popular means to gain insights into a software project's evolution, monitor project health, support decisions and derive best practices. Tools supporting the mining process are commonly applied by researchers and practitioners, but their limitations and agreement are often not well understood. Objective: This study investigates some threats to validity in complex tool pipelines for evolutionary software analyses and evaluates the tools' agreement in terms of data, study outcomes and conclusions for the same research questions. Method: We conduct a lightweight literature review to select three studies on collaboration and coordination, software maintenance and software quality from high-ranked venues, which we formally replicate with four independent, systematically selected mining tools to quantitatively and qualitatively compare the extracted data, analysis results and conclusions. Results: We find that numerous technical details in tool design and implementation accumulate along the complex mining pipelines and can cause substantial differences in the extracted baseline data, its derivatives, subsequent results of statistical analyses and, under specific circumstances, conclusions. Conclusions: Users must carefully choose tools and evaluate their limitations to assess the scope of validity in an adequate way. Reusing tools is recommended. Researchers and tool authors can promote reusability and help reducing uncertainties by reproduction packages and comparative studies following our approach.}, language = {en} }