@article{e5348fc8a1b8489aa48421197e710d9b,
title = "Multi-center validation of an artificial intelligence system for detection of COVID-19 on chest radiographs in symptomatic patients",
abstract = "Objectives: While chest radiograph (CXR) is the first-line imaging investigation in patients with respiratory symptoms, differentiating COVID-19 from other respiratory infections on CXR remains challenging. We developed and validated an AI system for COVID-19 detection on presenting CXR. Methods: A deep learning model (RadGenX), trained on 168,850 CXRs, was validated on a large international test set of presenting CXRs of symptomatic patients from 9 study sites (US, Italy, and Hong Kong SAR) and 2 public datasets from the US and Europe. Performance was measured by area under the receiver operator characteristic curve (AUC). Bootstrapped simulations were performed to assess performance across a range of potential COVID-19 disease prevalence values (3.33 to 33.3%). Comparison against international radiologists was performed on an independent test set of 852 cases. Results: RadGenX achieved an AUC of 0.89 on 4-fold cross-validation and an AUC of 0.79 (95%CI 0.78–0.80) on an independent test cohort of 5,894 patients. Delong{\textquoteright}s test showed statistical differences in model performance across patients from different regions (p < 0.01), disease severity (p < 0.001), gender (p < 0.001), and age (p = 0.03). Prevalence simulations showed the negative predictive value increases from 86.1% at 33.3% prevalence, to greater than 98.5% at any prevalence below 4.5%. Compared with radiologists, McNemar{\textquoteright}s test showed the model has higher sensitivity (p < 0.001) but lower specificity (p < 0.001). Conclusion: An AI model that predicts COVID-19 infection on CXR in symptomatic patients was validated on a large international cohort providing valuable context on testing and performance expectations for AI systems that perform COVID-19 prediction on CXR. Key Points: • An AI model developed using CXRs to detect COVID-19 was validated in a large multi-center cohort of 5,894 patients from 9 prospectively recruited sites and 2 public datasets. • Differences in AI model performance were seen across region, disease severity, gender, and age. • Prevalence simulations on the international test set demonstrate the model{\textquoteright}s NPV is greater than 98.5% at any prevalence below 4.5%.",
keywords = "Artificial intelligence, COVID-19, Public health, Radiology, Thoracic",
author = "Kuo, {Michael D.} and Chiu, {Keith W.H.} and Wang, {David S.} and Larici, {Anna Rita} and Dmytro Poplavskiy and Adele Valentini and Alessandro Napoli and Andrea Borghesi and Guido Ligabue and Fang, {Xin Hao B.} and Wong, {Hing Ki C.} and Sailong Zhang and Hunter, {John R.} and Abeer Mousa and Amato Infante and Lorenzo Elia and Salvatore Golemi and Yu, {Leung Ho P.} and Hui, {Christopher K.M.} and Erickson, {Bradley J.}",
note = "Funding Information: The following authors have also contributed to the manuscript: Andrea Leonardi (MD)1, Carlo Catalano (MD)1, Paolo Ricci (MD)1, Hiu Yin S. Lam (MBBS)2, Ho Yuen F. Wong (MBBS)2, Gilbert Lui (PhD)3, Nicoletta Izzi1 (MD)4, Antonella Donatelli (MD)4, Francesca Marchetti (MD)4, Annie Rhee (MD)5, Lorenzo Preda (MD)6, Nicola Carapella (MD)7, Helen Zhi (PhD)8, Francesco Ascari (MD)9,10, Patrizia Lazzari (MD)9,10, Leonardo Canulli (MD)9,11, Pietro Torricelli (MD)9,10, San Ming P. Yu (MBBS)12, Yu Wai T. Hon (MBBS)12, Yee Hing J. Hui (MBBS)12, Cesare Colosimo (MD)13,14, Luigi Natale (MD)13,14, Riccardo Marano (MD)13,14, Maurizio Sanguinetti (MD)13,15Affiliations:1Department of Radiological, Oncological and Pathological Sciences, Sapienza University of Rome, Rome; Italy2Radiology Department, Queen Mary Hospital; Hong Kong SAR, China3Department of Medicine, LKS Faculty of Medicine, The University of Hong Kong; Hong Kong SAR, China4Department of Clinical, Surgical, Diagnostic and Pediatric Sciences, University of Pavia; Italy5Radiology Department, Mayo Clinic; Rochester, Minnesota, USA6Department of Radiology, Fondazione IRCCS Policlinico San Matteo; Italy7Department of Medical and Surgical Specialties, Radiological Sciences and Public Health, University of Brescia, ASST Spedali Civili of Brescia; Italy8School of Public Health, LKS Faculty of Medicine, The University of Hong Kong; Hong Kong SAR, China9Department of Medical and Surgical Sciences for Children & Adults, Modena and Reggio Emilia University; Italy10Division of Radiology, Azienda Ospedaliero-Universitaria Policlinico di Modena; Italy11Division of Education, Research and Innovation, Azienda Ospedaliero-Universitaria Policlinico di Modena; Italy12Radiology Department, United Christian Hospital; Hong Kong SAR, China13Department of Radiological and Hematological Sciences, Section of Radiology, Universit{\`a} Cattolica del Sacro Cuore; Rome, Italy14Department of Diagnostic Imaging, Oncological Radiotherapy and Hematology, Fondazione Policlinico Universitario “A. Gemelli” IRCCS; Rome, Italy15Institute of Microbiology and Division of Clinical Microbiology, Universit{\`a} Cattolica del Sacro Cuore, Rome; Italy Publisher Copyright: {\textcopyright} 2022, The Author(s), under exclusive licence to European Society of Radiology.",
year = "2023",
month = jan,
doi = "10.1007/s00330-022-08969-z",
language = "English (US)",
volume = "33",
pages = "23--33",
journal = "European radiology",
issn = "0938-7994",
publisher = "Springer Verlag",
number = "1",
}