@unpublished{GaubeJussupowKokjeetal., author = {Gaube, Susanne and Jussupow, Ekaterina and Kokje, Eesha and Khan, Jowaria and Bondi-Kelly, Elizabeth and Schicho, Andreas and Kitamura, Felipe Campos and Koch, Timo Kevin and Ezer, Timur and Mottok, J{\"u}rgen and Lermer, Eva and Ghassemi, Marzyeh and Colak, Errol}, title = {Underreliance Harms Human-AI Collaboration More Than Overreliance in Medical Imaging}, publisher = {OSFPreprints}, doi = {10.31219/osf.io/4wv8j}, abstract = {Importance: The use of artificial intelligence (AI) to support clinicians in diagnostic decision-making holds significant potential; however, evidence regarding its clinical utility remains mixed. In many cases, the interaction between healthcare professionals and AI systems does not improve collaborative performance compared to the standalone performance of humans or AI. Currently, the underlying mechanisms that limit human-AI collaboration are poorly understood. Objective: To examine the impact of AI advice on diagnostic decision-making among experts and novices, focusing on understanding the role of explainability (XAI) on users' reliance on advice. Design, Setting, and Participants: A mixed-methods design combining a crossover experimental design with a think-aloud and an eye-tracking study arm was conducted in 2023. Participants were task experts (radiologists) and novices (non-radiologist physicians and medical trainees) from 10 countries, with the think-aloud and eye-tracking conducted in Germany. Intervention: Participants reviewed 50 patient cases containing head CT scans and patient information. Every case was reviewed in three time-separate sessions in randomized order. In each session, participants were exposed to a different experimental condition: (a) control, i.e., no AI prediction presented; (b) basic advice, i.e., AI prediction without annotations; and (c) XAI advice, i.e., AI prediction with annotations. For each case, participants had to determine if the patients had an intracranial hemorrhage (ICH), rate their confidence, and, if applicable, the usefulness of the AI advice. Main Outcome(s) and Measure(s): Diagnostic performance, confidence in the diagnosis, case reading time, and AI advice usefulness ratings. Results: The data analysis included 125 participants. The mean age was 28.5 years (SD = 6.72), and 55.2\% identified as female. Underreliance on correct AI advice was associated with high uncertainty and had a more detrimental impact on diagnostic performance than overreliance on incorrect advice. XAI advice reduced underreliance and improved performance and confidence, particularly when reviewing more difficult cases with ICH. AI advice, particularly XAI, did not reduce reading time. XAI was perceived as more useful than basic AI advice, especially among novices. Conclusions and Relevance: Our findings indicate that underreliance on AI might be more harmful than overreliance, highlighting the need to develop efficient counterstrategies beyond current XAI methods.}, language = {en} } @unpublished{GaubeJussupowKokjeetal., author = {Gaube, Susanne and Jussupow, Ekaterina and Kokje, Eesha and Khan, Jowaria and Bondi-Kelly, Elizabeth and Schicho, Andreas and Kitamura, Felipe Campos and Koch, Timo Kevin and Ezer, Timur and Mottok, J{\"u}rgen and Lermer, Eva and Ghassemi, Marzyeh and Colak, Errol}, title = {Examining Reliance Patterns on AI Advice in Medical Imaging: a Mixed-Methods Randomized Crossover Experiment}, doi = {10.31219/osf.io/4wv8j_v3}, pages = {20}, abstract = {Background: Artificial intelligence (AI) holds significant potential to support diagnostic decision-making; however, evidence regarding its clinical utility remains mixed. Often, the collaboration between clinicians and AI systems does not surpass the individual performance of unaided humans or standalone AI. Yet, currently, the mechanisms that limit human-AI synergy are poorly understood. This study examined the impact of AI advice on diagnostic decision-making among experts and novices, focusing on reliance patterns. Methods: We used a mixed-methods crossover experimental design with a think-aloud and an eye-tracking study arm. Participants were 50 task experts (radiologists) and 75 novices (non-radiologist physicians and medical trainees) from 10 countries. They reviewed 50 head CT scans and every case was examined in three time-separate sessions in randomized order. In each session, participants were exposed to different experimental conditions: (a) control, no AI prediction; (b) basic advice, AI prediction without annotations; and (c) XAI advice, AI prediction with scan annotations. For each case, participants had to determine if the patients had an intracranial hemorrhage (ICH). The main outcomes were diagnostic performance, confidence in the diagnosis, case reading time, and AI advice usefulness ratings. Findings: Both overreliance on incorrect advice and underreliance on correct advice occurred. Underreliance was associated with high uncertainty and, in absolute terms, had a more detrimental impact on diagnostic performance than overreliance. Correct XAI advice reduced underreliance, improved performance (OR=1·84, p<0·0001), and confidence (b=0·15, p<0·0001), particularly when reviewing more difficult cases with ICH. Surprisingly, correct XAI did not reduce reading time (b=1·81, p=0·0713). XAI was perceived as more useful than basic AI advice (b=0·12, p=0·0029), especially among novices. Interpretation: The occurrence of both under- and overreliance highlights the need for efficient counterstrategies beyond classic XAI methods to foster appropriate reliance and synergy.}, language = {en} }