[{"id": 568410, "type": "GenericWebPage", "mime_type": "text/html", "url": "https://mdrregulator.com/news/the-us-fda-updated-the-list-of-ai-ml-enabled-medical-devices-authorizing-950-devices", "slug": "mdrregulator", "title": "The US FDA updated the list of AI/ML-Enabled Medical Devices, authorizing 950 devices \u2695\ufe0f MDR Regulator", "snippet": "The FDA\u2019s updated list of AI/ML-enabled medical devices, which now includes 950 authorized devices, underscores the rapid growth in this field. The 37% increase from last year\u2019s total of 692 devices is indicative of the expanding role of AI/ML in healthcare.", "image": null, "date_published": null, "media": [], "products": [], "source": {"id": 568410, "website_id": 125860, "url": "https://mdrregulator.com/news/the-us-fda-updated-the-list-of-ai-ml-enabled-medical-devices-authorizing-950-devices", "title": "The US FDA updated the list of AI/ML-Enabled Medical Devices, authorizing 950 devices \u2695\ufe0f MDR Regulator", "snippet": "The FDA\u2019s updated list of AI/ML-enabled medical devices, which now includes 950 authorized devices, underscores the rapid growth in this field. The 37% increase from last year\u2019s total of 692 devices is indicative of the expanding role of AI/ML in healthcare.", "cleaned_text": null, "cleaned_text_custom_parser": null, "cleaned_text_custom_parser_2": null, "best_field": null, "enhanced": false, "date_published": null, "image": null, "num_tokens_oai_tokenizer": null, "num_tokens_gemini_tokenizer": null}, "favicon": null, "state": "candidate", "enhanced": false, "answer": null, "valid_answer": false, "source_citation_index": 1, "user_web_page_id": null, "status": "not_processed", "reader": false, "rank": 1, "reason": "The source provides a list of FDA-authorized AI-enabled medical devices, making it very relevant to the query for official database references."}, {"id": 568405, "type": "GenericWebPage", "mime_type": "text/html", "url": "https://www.medtechspectrum.com/analysis/12/24541/the-2025-index-100-fda-approved-ai-driven-medical-devices.html", "slug": "medtechspectrum", "title": "The 2025 Index: 100 FDA-Approved AI-Driven Medical Devices", "snippet": "The U.S. Food and Drug Administration (FDA) has recently authorised over 100 artificial intelligence (AI)-enabled medical devices across a broad range of clinical applications, reflecting the accelerating integration of machine learning and software-based diagnostics into healthcare.", "image": null, "date_published": null, "media": [], "products": [], "source": {"id": 568405, "website_id": 125859, "url": "https://www.medtechspectrum.com/analysis/12/24541/the-2025-index-100-fda-approved-ai-driven-medical-devices.html", "title": "The 2025 Index: 100 FDA-Approved AI-Driven Medical Devices", "snippet": "The U.S. Food and Drug Administration (FDA) has recently authorised over 100 artificial intelligence (AI)-enabled medical devices across a broad range of clinical applications, reflecting the accelerating integration of machine learning and software-based diagnostics into healthcare.", "cleaned_text": null, "cleaned_text_custom_parser": null, "cleaned_text_custom_parser_2": null, "best_field": null, "enhanced": false, "date_published": null, "image": null, "num_tokens_oai_tokenizer": null, "num_tokens_gemini_tokenizer": null}, "favicon": null, "state": "candidate", "enhanced": false, "answer": null, "valid_answer": false, "source_citation_index": 2, "user_web_page_id": null, "status": "not_processed", "reader": false, "rank": 1, "reason": "The source highlights the number of AI medical devices recently authorized by the FDA and discusses their significance in clinical settings."}, {"id": 543772, "type": "GenericWebPage", "mime_type": "text/html", "url": "https://www.medtechdive.com/news/5-takeaways-from-the-fdas-updated-list-of-cleared-aiml-medical-devices/697570/", "slug": "medtechdive", "title": "5 takeaways from the FDA\u2019s updated list of cleared AI/ML medical devices", "snippet": "The vast majority of AI/ML devices cleared between August 2022 and July 2023 went through the FDA\u2019s 510(k) pathway. Just two devices received de novo clearance during that time period.", "image": null, "date_published": null, "media": [], "products": [], "source": {"id": 543772, "website_id": 9519, "url": "https://www.medtechdive.com/news/5-takeaways-from-the-fdas-updated-list-of-cleared-aiml-medical-devices/697570/", "title": "5 takeaways from the FDA\u2019s updated list of cleared AI/ML medical devices", "snippet": "The vast majority of AI/ML devices cleared between August 2022 and July 2023 went through the FDA\u2019s 510(k) pathway. Just two devices received de novo clearance during that time period.", "cleaned_text": "A growing number of medical devices include artificial intelligence or machine learning features, and the Food and Drug Administration has started tracking ones that have gone through the agency\u2019s approval. The FDA recently updated its list of AI/ML-enabled devices, showing that nearly 700 have now received marketing authorization.\n\nThe latest update added a total of 171 new devices, the majority of which were cleared between August 2022 and July 2023.\n\nLast year, when the FDA first unveiled the list, MedTech Dive found that the majority of cleared AI/ML devices were used in radiology and went through the agency\u2019s 510(k) clearance pathway. That continued to be true, based on an analysis of the newest additions.\n\nHere are five takeaways from the updated list:\n\n1. The rate of FDA submissions slowed in 2021 and 2022 but has started to pick up again\n\nThe number of AI/ML device submissions the FDA has received has increased every year since 2015. The agency saw the biggest increase between 2019 and 2020, when the number of submissions increased by 39%. The rate of submissions slowed in 2021 and 2022, but is expected to surpass 30% again this year, the agency said in a summary provided with the list.\n\n2. GE HealthCare had the most cleared AI/ML devices\n\nGE HealthCare and Siemens Healthineers led the list with the most cleared AI/ML devices between August 2022 and July 2023, according to an analysis by MedTech Dive. GE HealthCare has had the most authorized AI/ML devices to date.\n\nOther companies that topped the list include Aidoc, a Tel Aviv-based company that makes triage systems for radiologists, and Brainlab, a Munich-based firm partnering with ZimVie on surgical navigation for spine procedures.\n\n3. Radiology still accounts for the majority of AI/ML devices\n\nLast year, MedTech Dive found that the majority of AI/ML devices authorized to date have been in the field of radiology. That trend has continued, with radiology accounting for 87% of new devices last year, and 79% of new devices in the first half of 2023.\n\nPart of the reason radiology might be so represented on the list is because of the early digitization of imaging data compared to other specialties. The 21st Century Cures Act also specifies anything that analyzes a medical image should be regulated as a medical device. The FDA sought to clarify through a final guidance last year that certain types of software should be regulated as devices, such as those that use information from electronic medical records to flag potential cases of sepsis.\n\n4. None of the devices on the list used generative AI\n\nDespite rising interest in large language models, such as ChatGPT, this type of technology was not used in any of the cleared devices. As of Oct. 19, no device has been authorized that uses generative AI or artificial general intelligence or is powered by large language models, the FDA said.\n\nThe types of models featured in the list vary in complexity. Some were \u201cshallow,\u201d having less than two hidden layers between the algorithm\u2019s input and output, while others used deep learning, the FDA said. Typically, the models used a combination of approaches to achieve a result, such as using one model to generate features and another model to do classification.\n\nSome examples of the types of devices cleared by the FDA this year include an algorithm developed by Edwards Lifesciences to help clinicians predict the risk of a global hypoperfusion event in patients receiving advanced hemodynamic monitoring, and a radiation planning assistant developed by MD Anderson Cancer Center to help plan cancer treatment.\n\n5. Most devices still went through the 510(k) pathway\n\nThe vast majority of AI/ML devices cleared between August 2022 and July 2023 went through the FDA\u2019s 510(k) pathway. Just two devices received de novo clearance during that time period. This is consistent with previous years: 96% of cleared AI/ML devices went through the 510(k) pathway between 1995 and 2022, according to an analysis by MedTech Dive last year.", "cleaned_text_custom_parser": "Don't miss tomorrow's medtech industry news\n5 takeaways from the FDA\u2019s updated list of cleared AI/ML medical devices\nThe agency recently added 171 new devices to its list, with radiology once again accounting for the most authorizations.\nA growing number of medical devices include artificial intelligence or machine learning features, and the Food and Drug Administration has started tracking ones that have gone through the agency\u2019s approval. The FDA recentlyupdated its list of AI/ML-enabled devices, showing that nearly 700 have now received marketing authorization.\nThe latest update added a total of 171 new devices, the majority of which were cleared between August 2022 and July 2023.\nLast year, when the FDA first unveiled the list, MedTech Dive found that themajority of cleared AI/ML deviceswere used in radiology and went through the agency\u2019s 510(k) clearance pathway. That continued to be true, based on an analysis of the newest additions.\n1. The rate of FDA submissions slowed in 2021 and 2022 but has started to pick up again\nThe number of AI/ML device submissions the FDA has received has increased every year since 2015. The agency saw the biggest increase between 2019 and 2020, when the number of submissions increased by 39%. The rate of submissions slowed in 2021 and 2022, but is expected to surpass 30% again this year, the agency said in asummary provided with the list.\n2. GE HealthCare had the most cleared AI/ML devices\nGE HealthCare and SiemensHealthineersled the list with the most cleared AI/ML devices between August 2022 and July 2023, according to an analysis byMedTechDive. GE HealthCare has had the most authorized AI/ML devices to date.\nOther companies that topped the list includeAidoc, a Tel Aviv-based company that makes triage systems for radiologists, andBrainlab, a Munich-based firmpartnering with ZimVieon surgical navigation for spine procedures.\n3. Radiology still accounts for the majority of AI/ML devices\nLast year, MedTech Dive found that the majority of AI/ML devices authorized to date have been in the field of radiology. That trend has continued, with radiology accounting for 87% of new devices last year, and 79% of new devices in the first half of 2023.\nPart of the reason radiology might be so represented on the list is because of the early digitization of imaging data compared to other specialties.\u00a0The 21st Century Cures Act also specifies anything that analyzes a medical image should be regulated as a medical device. The FDA sought to clarify through afinal guidance last yearthat certain types of software should be regulated as devices, such as those that use information from electronic medical records to flag potential cases of sepsis.\n4. None of the devices on the list used generative AI\nDespite rising interest in large language models, such as ChatGPT, this type of technology was not used in any of the cleared devices. As of Oct.\u00a019, no device has been authorized that uses generative AI or artificial general intelligence or is powered by large language models, the FDA said.\nThe types of models featured in the list vary in complexity. Some were \u201cshallow,\u201d having less than two hidden layers between the algorithm\u2019s input and output, while others used deep learning, the FDA said. Typically, the models used a combination of approaches to achieve a result, such as using one model to generate features and another model to do classification.\nSome examples of the types of devices cleared by the FDA this year include an algorithm developed by Edwards Lifesciences to help clinicianspredict the risk of a global hypoperfusion eventin patients receiving advanced hemodynamic monitoring, and aradiation planning assistantdeveloped by MD Anderson Cancer Center to help plan cancer treatment.\n5. Most devices still went through the 510(k) pathway\nThe vast majority of AI/ML devices cleared between August 2022 and July 2023 went through the FDA\u2019s 510(k) pathway. Just two devices received de novo clearance during that time period. This is consistent with previous years:\u00a096% of cleared AI/ML devices went through the 510(k) pathway between 1995 and 2022, according to ananalysis by MedTech Dive last year.\nRecommended Reading\nMedTech Dive news delivered to your inbox\nEditors' picks\nRoundup: Medtech M&A slump continues but analysts predict uptick in 2024\nGlobus Medical and Thermo Fisher were among the companies to announce billion dollar deals this year, while Medtronic and\u00a0Boston Scientific called off acquisitions.\nRoundup: Top medtech companies cut jobs in 2023\nJohnson & Johnson, Medtronic and Abbott were among the top companies to announce layoffs this year as the industry implemented cost-cutting measures.\nGet the free newsletter\nMost Popular\nLibrary resources\nCompany Announcements\nWhat We're Reading\nEvents\nMedTech Dive news delivered to your inbox\nCompany Announcements\nRoundup: Medtech M&A slump continues but analysts predict uptick in 2024\nGlobus Medical and Thermo Fisher were among the companies to announce billion dollar deals this year, while Medtronic and\u00a0Boston Scientific called off acquisitions.\nRoundup: Top medtech companies cut jobs in 2023\nJohnson & Johnson, Medtronic and Abbott were among the top companies to announce layoffs this year as the industry implemented cost-cutting measures.\nExplore\nReach our audience\nRelated Publications\nGet MedTech Dive in your inbox", "cleaned_text_custom_parser_2": "\n\n \n \n Skip to main content\n \n

\n CONTINUE TO SITE \u279e\n

\n \n

\n An article from\n $\"site$ \n

\n 5 takeaways from the FDA\u2019s updated list of cleared AI/ML medical devices\n

\n The agency recently added 171 new devices to its list, with radiology once again accounting for the most authorizations.\n

\n Published Oct. 24, 2023\n

\n \n Elise Reuter\n \n Senior Reporter\n

\n GE HealthCare's Revolution Ascend CT System includes an AI feature that reduces the number of clicks necessary to execute a CT exam by 66%.\n Courtesy of GE Healthcare\n

\n \n

\n A growing number of medical devices include artificial intelligence or machine learning features, and the Food and Drug Administration has started tracking ones that have gone through the agency\u2019s approval. The FDA recently\n \n updated its list of AI/ML-enabled devices\n \n , showing that nearly 700 have now received marketing authorization.\n

\n The latest update added a total of 171 new devices, the majority of which were cleared between August 2022 and July 2023.\n

\n Last year, when the FDA first unveiled the list, MedTech Dive found that the\n \n majority of cleared AI/ML devices\n \n were used in radiology and went through the agency\u2019s 510(k) clearance pathway. That continued to be true, based on an analysis of the newest additions.\n

\n Here are five takeaways from the updated list:\n

\n 1. The rate of FDA submissions slowed in 2021 and 2022 but has started to pick up again\n

\n The number of AI/ML device submissions the FDA has received has increased every year since 2015. The agency saw the biggest increase between 2019 and 2020, when the number of submissions increased by 39%. The rate of submissions slowed in 2021 and 2022, but is expected to surpass 30% again this year, the agency said in a\n \n summary provided with the list.\n \n

\n 2. GE HealthCare had the most cleared AI/ML devices\n

\n GE HealthCare and Siemens\n Healthineers\n led the list with the most cleared AI/ML devices between August 2022 and July 2023, according to an analysis by\n MedTech\n Dive. GE HealthCare has had the most authorized AI/ML devices to date.\n

\n Other companies that topped the list include\n Aidoc\n , a Tel Aviv-based company that makes triage systems for radiologists, and\n Brainlab\n , a Munich-based firm\n \n partnering with ZimVie\n \n on surgical navigation for spine procedures.\n

\n 3. Radiology still accounts for the majority of AI/ML devices\n

\n Last year, MedTech Dive found that the majority of AI/ML devices authorized to date have been in the field of radiology. That trend has continued, with radiology accounting for 87% of new devices last year, and 79% of new devices in the first half of 2023.\n

\n Part of the reason radiology might be so represented on the list is because of the early digitization of imaging data compared to other specialties.\u00a0The 21st Century Cures Act also specifies anything that analyzes a medical image should be regulated as a medical device. The FDA sought to clarify through a\n \n final guidance last year\n \n that certain types of software should be regulated as devices, such as those that use information from electronic medical records to flag potential cases of sepsis.\n

\n 4. None of the devices on the list used generative AI\n

\n Despite rising interest in large language models, such as ChatGPT, this type of technology was not used in any of the cleared devices. As of Oct.\u00a019, no device has been authorized that uses generative AI or artificial general intelligence or is powered by large language models, the FDA said.\n

\n The types of models featured in the list vary in complexity. Some were \u201cshallow,\u201d having less than two hidden layers between the algorithm\u2019s input and output, while others used deep learning, the FDA said. Typically, the models used a combination of approaches to achieve a result, such as using one model to generate features and another model to do classification.\n

\n Some examples of the types of devices cleared by the FDA this year include an algorithm developed by Edwards Lifesciences to help clinicians\n \n predict the risk of a global hypoperfusion event\n \n in patients receiving advanced hemodynamic monitoring, and a\n \n radiation planning assistant\n \n developed by MD Anderson Cancer Center to help plan cancer treatment.\n

\n 5. Most devices still went through the 510(k) pathway\n

\n The vast majority of AI/ML devices cleared between August 2022 and July 2023 went through the FDA\u2019s 510(k) pathway. Just two devices received de novo clearance during that time period. This is consistent with previous years:\u00a096% of cleared AI/ML devices went through the 510(k) pathway between 1995 and 2022, according to an\n \n analysis by MedTech Dive last year.\n \n

\n Recommended Reading\n

\n \n 5 takeaways from the FDA\u2019s list of AI-enabled medical devices\n \n \n \n By\n \n Elise Reuter\n \n \u2022\n \n Nov. 7, 2022\n \n
\n \n Google expands AI mammography deal with iCAD\n \n \n By\n \n \n \n Emily Olsen\n \n \n \u2022\n Aug. 18, 2023\n \n

\n \n $\"License$ \n purchase licensing rights\n \n

\n Filed Under:\n \n Digital Health,\n \n \n Medical Devices,\n \n \n FDA\n \n

\n MedTech Dive news delivered to your inbox\n

\n Get the free daily newsletter read by industry experts\n

\n \n

\n Editors' picks\n

\n
\n
\n $\"Image$ \n Courtesy of Thermo Fisher Scientific\n $\"Image$ \n
\n
\n
\n
\n \n Roundup: Medtech M&A slump continues but analysts predict uptick in 2024\n \n
\n
\n Globus Medical and Thermo Fisher were among the companies to announce billion dollar deals this year, while Medtronic and\u00a0Boston Scientific called off acquisitions.\n
\n
\n By Nick Paul Taylor \u2022\n Dec. 19, 2023\n
\n
\n
\n
\n
\n $\"Image$ \n Stock via Getty Images\n $\"Image$ \n
\n
\n
\n
\n \n Roundup: Top medtech companies cut jobs in 2023\n \n
\n
\n Johnson & Johnson, Medtronic and Abbott were among the top companies to announce layoffs this year as the industry implemented cost-cutting measures.\n
\n
\n By Ricky Zipp \u2022\n Nov. 27, 2023\n
\n
\n

\n Keep up with the story. Subscribe to the MedTech Dive free daily newsletter\n

\n \n

\n MedTech Dive news delivered to your inbox\n

\n Get the free daily newsletter read by industry experts\n

\n \n

\n Company Announcements\n

\n \n View all\n \n |\n \n Post a press release\n \n

\n \n

\n Dornier MedTech America launches Hoover\u2122 and AXIS\u00ae II SLIM, raising the bar in kidney stone tr\u2026\n

\n From Dornier MedTech America (DMTA) part of Advanced MedTech\n September 17, 2025\n

\n \n \n

\n Three Overlooked Evidence Engines Could Give MedTechs 2-Year Market Advantage\n

\n From MedTech Leading Voice - illumino.co\n September 15, 2025\n

\n \n \n

\n Femasys Closes $8 Million in Financing to Accelerate Commercialization of Fertility and Birth \u2026\n

\n From Femasys Inc.\n September 10, 2025\n

\n \n \n

\n Checkpoint Surgical Marks Milestone with First Use of Investigational Nerve Regeneration Syste\u2026\n

\n From Checkpoint Surgical, Inc.\n September 08, 2025\n

\n \n

\n Editors' picks\n

\n
\n
\n $\"Image$ \n Courtesy of Thermo Fisher Scientific\n $\"Image$ \n
\n
\n
\n
\n \n Roundup: Medtech M&A slump continues but analysts predict uptick in 2024\n \n
\n
\n Globus Medical and Thermo Fisher were among the companies to announce billion dollar deals this year, while Medtronic and\u00a0Boston Scientific called off acquisitions.\n
\n
\n By Nick Paul Taylor \u2022\n Dec. 19, 2023\n
\n
\n
\n
\n
\n $\"Image$ \n Stock via Getty Images\n $\"Image$ \n
\n
\n
\n
\n \n Roundup: Top medtech companies cut jobs in 2023\n \n
\n
\n Johnson & Johnson, Medtronic and Abbott were among the top companies to announce layoffs this year as the industry implemented cost-cutting measures.\n
\n
\n By Ricky Zipp \u2022\n Nov. 27, 2023\n
\n
\n

\n Latest in Digital Health\n

\n
\n \n FDA warns on unauthorized blood pressure, infant monitoring devices\n \n By Elise Reuter\n
\n
\n
\n \n GOP bill extends telehealth flexibilities, sidesteps ACA subsidies\n \n By Emily Olsen\n
\n
\n
\n \n How three companies are using foundation models in radiology\n \n By Elise Reuter\n
\n
\n
\n \n Foundation models are medtech\u2019s newest trend. But what are they?\n \n By Elise Reuter\n
\n

\n Industry Dive is an Informa TechTarget business.\n

\n \u00a9 2025 TechTarget, Inc. or its subsidiaries. All rights reserved. | View our\n \n other publications\n \n |\n \n Privacy policy\n \n |\n \n Terms of use\n \n |\n \n Take down policy\n \n .\n
\n \n Cookie Preferences / Do Not Sell\n \n

\n This website is owned and operated by Informa TechTarget, part of a global network that informs, influences and connects the world's technology buyers and sellers. All copyright resides with them. Informa PLC's registered office is 5 Howick Place, London SW1P 1WG. Registered in England and Wales. TechTarget, Inc.'s registered office is 275 Grove St. Newton, MA 02466.\n

\n \n\n", "best_field": "cleaned_text", "enhanced": true, "date_published": null, "image": null, "num_tokens_oai_tokenizer": null, "num_tokens_gemini_tokenizer": null}, "favicon": "https://www.medtechdive.com/favicon.ico?v=3", "state": "candidate", "enhanced": true, "answer": null, "valid_answer": false, "source_citation_index": 3, "user_web_page_id": null, "status": "ready", "reader": false, "rank": 1, "reason": "The source offers critical insights into the updates and clearances of AI/ML-enabled medical devices issued by the FDA."}, {"id": 568431, "type": "GenericWebPage", "mime_type": "text/html", "url": "https://healthimaging.com/topics/healthcare-management/healthcare-policy/fda-adds-dozens-ai-enabled-radiology-applications-list-clearances", "slug": "healthimaging", "title": "FDA adds dozens of AI-enabled radiology applications to list of clearances", "snippet": "Around two-thirds of all approved artificial intelligence-powered clinical devices are catered to radiology settings.", "image": null, "date_published": null, "media": [], "products": [], "source": {"id": 568431, "website_id": 125862, "url": "https://healthimaging.com/topics/healthcare-management/healthcare-policy/fda-adds-dozens-ai-enabled-radiology-applications-list-clearances", "title": "FDA adds dozens of AI-enabled radiology applications to list of clearances", "snippet": "Around two-thirds of all approved artificial intelligence-powered clinical devices are catered to radiology settings.", "cleaned_text": null, "cleaned_text_custom_parser": null, "cleaned_text_custom_parser_2": null, "best_field": null, "enhanced": false, "date_published": null, "image": null, "num_tokens_oai_tokenizer": null, "num_tokens_gemini_tokenizer": null}, "favicon": null, "state": "candidate", "enhanced": false, "answer": null, "valid_answer": false, "source_citation_index": 4, "user_web_page_id": null, "status": "not_processed", "reader": false, "rank": 2, "reason": "This source discusses FDA updates specifically regarding AI-enabled medical devices focused on radiology, making it relevant to practitioners."}, {"id": 568440, "type": "GenericWebPage", "mime_type": "text/html", "url": "https://med.stanford.edu/news/all-news/2025/06/chatehr.html", "slug": "stanford", "title": "Clinicians can \u2018chat\u2019 with medical records through new AI software, ChatEHR", "snippet": "ChatEHR, artificial intelligence software developed at Stanford Medicine, is expediting chart reviews and other tasks by allowing clinicians to ask questions of medical records.", "image": null, "date_published": null, "media": [], "products": [], "source": {"id": 568440, "website_id": 3508, "url": "https://med.stanford.edu/news/all-news/2025/06/chatehr.html", "title": "Clinicians can \u2018chat\u2019 with medical records through new AI software, ChatEHR", "snippet": "ChatEHR, artificial intelligence software developed at Stanford Medicine, is expediting chart reviews and other tasks by allowing clinicians to ask questions of medical records.", "cleaned_text": null, "cleaned_text_custom_parser": null, "cleaned_text_custom_parser_2": null, "best_field": null, "enhanced": false, "date_published": null, "image": null, "num_tokens_oai_tokenizer": null, "num_tokens_gemini_tokenizer": null}, "favicon": "https://med.stanford.edu/etc/clientlibs/sm/images/SM-Shield-152x152.png", "state": "candidate", "enhanced": false, "answer": null, "valid_answer": false, "source_citation_index": 5, "user_web_page_id": null, "status": "not_processed", "reader": false, "rank": 2, "reason": "The source discusses the new approvals and provides context on current trends in FDA authorizations of AI-enabled devices."}, {"id": 568428, "type": "GenericWebPage", "mime_type": "text/html", "url": "https://www.dlapiper.com/en-us/insights/publications/2025/01/fda-issues-artificial-intelligence-enabled-device-software-functions-guidance", "slug": "dlapiper", "title": "FDA issues Artificial Intelligence-Enabled Device Software Functions draft guidance | DLA Piper", "snippet": "While the guidance does not provide specifics on how these risk analysis programs and documentation or monitoring could differ from traditional devices, FDA notes that details on the submitter\u2019s post-market quality system regulation compliance plan (eg, performance monitoring plans) may be ...", "image": null, "date_published": null, "media": [], "products": [], "source": {"id": 568428, "website_id": 27951, "url": "https://www.dlapiper.com/en-us/insights/publications/2025/01/fda-issues-artificial-intelligence-enabled-device-software-functions-guidance", "title": "FDA issues Artificial Intelligence-Enabled Device Software Functions draft guidance | DLA Piper", "snippet": "While the guidance does not provide specifics on how these risk analysis programs and documentation or monitoring could differ from traditional devices, FDA notes that details on the submitter\u2019s post-market quality system regulation compliance plan (eg, performance monitoring plans) may be ...", "cleaned_text": null, "cleaned_text_custom_parser": null, "cleaned_text_custom_parser_2": null, "best_field": null, "enhanced": false, "date_published": null, "image": null, "num_tokens_oai_tokenizer": null, "num_tokens_gemini_tokenizer": null}, "favicon": "https://www.dlapiper.com/-/media/foundation/brand-fav-icon/favicon.ico", "state": "candidate", "enhanced": false, "answer": null, "valid_answer": false, "source_citation_index": 6, "user_web_page_id": null, "status": "not_processed", "reader": false, "rank": 2, "reason": "The source provides an overview of FDA's draft guidance relevant to marketing and managing AI-enabled devices."}, {"id": 543746, "type": "GenericWebPage", "mime_type": "text/html", "url": "https://www.thelancet.com/journals/landig/article/PIIS2589-7500(23)00126-7/fulltext", "slug": "thelancet", "title": "FDA-cleared artificial intelligence and machine learning-based medical devices and their 510(k) predicate networks - The Lancet Digital Health", "snippet": "The US Food and Drug Administration is clearing an increasing number of artificial intelligence and machine learning (AI/ML)-based medical devices through the 510(k) pathway. This pathway allows clearance if the device is substantially equivalent to a former cleared device (ie, predicate).", "image": null, "date_published": null, "media": [], "products": [], "source": {"id": 543746, "website_id": 7106, "url": "https://www.thelancet.com/journals/landig/article/PIIS2589-7500(23)00126-7/fulltext", "title": "FDA-cleared artificial intelligence and machine learning-based medical devices and their 510(k) predicate networks - The Lancet Digital Health", "snippet": "The US Food and Drug Administration is clearing an increasing number of artificial intelligence and machine learning (AI/ML)-based medical devices through the 510(k) pathway. This pathway allows clearance if the device is substantially equivalent to a former cleared device (ie, predicate).", "cleaned_text": null, "cleaned_text_custom_parser": null, "cleaned_text_custom_parser_2": null, "best_field": null, "enhanced": false, "date_published": null, "image": null, "num_tokens_oai_tokenizer": null, "num_tokens_gemini_tokenizer": null}, "favicon": null, "state": "candidate", "enhanced": false, "answer": null, "valid_answer": false, "source_citation_index": 7, "user_web_page_id": null, "status": "not_processed", "reader": false, "rank": 2, "reason": "While this source reviews the overall regulatory landscape for AI medical devices, it is important but less directly focused on the actual list of devices."}, {"id": 543748, "type": "GenericWebPage", "mime_type": "text/html", "url": "https://www.fda.gov/news-events/press-announcements/fda-issues-comprehensive-draft-guidance-developers-artificial-intelligence-enabled-medical-devices", "slug": "fda", "title": "FDA Issues Comprehensive Draft Guidance for Developers of Artificial Intelligence-Enabled Medical Devices | FDA", "snippet": "Today, the U.S. Food and Drug Administration issued draft guidance that includes recommendations to support development and marketing of safe and effective AI-enabled devices throughout the device\u2019s Total Product Life Cycle.", "image": null, "date_published": null, "media": [], "products": [], "source": {"id": 543748, "website_id": 1342, "url": "https://www.fda.gov/news-events/press-announcements/fda-issues-comprehensive-draft-guidance-developers-artificial-intelligence-enabled-medical-devices", "title": "FDA Issues Comprehensive Draft Guidance for Developers of Artificial Intelligence-Enabled Medical Devices | FDA", "snippet": "Today, the U.S. Food and Drug Administration issued draft guidance that includes recommendations to support development and marketing of safe and effective AI-enabled devices throughout the device\u2019s Total Product Life Cycle.", "cleaned_text": null, "cleaned_text_custom_parser": null, "cleaned_text_custom_parser_2": null, "best_field": null, "enhanced": false, "date_published": null, "image": null, "num_tokens_oai_tokenizer": null, "num_tokens_gemini_tokenizer": null}, "favicon": null, "state": "candidate", "enhanced": false, "answer": null, "valid_answer": false, "source_citation_index": 8, "user_web_page_id": null, "status": "not_processed", "reader": false, "rank": 2, "reason": "This source provides an overview of the general use of FDA-approved AI medical devices, which is relevant but less specific."}, {"id": 543764, "type": "GenericWebPage", "mime_type": "text/html", "url": "https://www.pew.org/en/research-and-analysis/issue-briefs/2021/08/how-fda-regulates-artificial-intelligence-in-medical-products", "slug": "pew", "title": "How FDA Regulates Artificial Intelligence in Medical Products", "snippet": "These devices must undergo the full premarket approval process, and developers must submit clinical evidence that the benefits of the product outweigh the risks.53 The continuous glucose monitoring system, Guardian Connect system, was approved through a premarket approval.54 \u00b7 Once a device ...", "image": null, "date_published": null, "media": [], "products": [], "source": {"id": 543764, "website_id": 102952, "url": "https://www.pew.org/en/research-and-analysis/issue-briefs/2021/08/how-fda-regulates-artificial-intelligence-in-medical-products", "title": "How FDA Regulates Artificial Intelligence in Medical Products", "snippet": "These devices must undergo the full premarket approval process, and developers must submit clinical evidence that the benefits of the product outweigh the risks.53 The continuous glucose monitoring system, Guardian Connect system, was approved through a premarket approval.54 \u00b7 Once a device ...", "cleaned_text": "Health care organizations are using artificial intelligence (AI)\u2014which the U.S. Food and Drug Administration defines as \u201cthe science and engineering of making intelligent machines\u201d\u2014for a growing range of clinical, administrative, and research purposes. This AI software can, for example, help health care providers diagnose diseases, monitor patients\u2019 health, or assist with rote functions such as scheduling patients.\n\nAlthough AI offers unique opportunities to improve health care and patient outcomes, it also comes with potential challenges. AI-enabled products, for example, have sometimes resulted in inaccurate, even potentially harmful, recommendations for treatment.1 These errors can be caused by unanticipated sources of bias in the information used to build or train the AI, inappropriate weight given to certain data points analyzed by the tool, and other flaws.\n\nThe regulatory framework governing these tools is complex. FDA regulates some\u2014but not all\u2014AI-enabled products used in health care, and the agency plays an important role in ensuring the safety and effectiveness of those products under its jurisdiction. The agency is currently considering how to adapt its review process for AI-enabled medical devices that have the ability to evolve rapidly in response to new data, sometimes in ways that are difficult to foresee.2\n\nThis brief describes current and potential uses of AI in health care settings and the challenges these technologies pose, outlines how and under what circumstances they are regulated by FDA, and highlights key questions that will need to be addressed to ensure that the benefits of these devices outweigh their risks. It will take a collective effort by FDA, Congress, technology developers, and the health care industry to ensure the safety and effectiveness of AI-enabled technology.\n\nWhat is AI and how is it used in health care?\n\nAI refers to the ability of a machine to perform a task that mimics human behavior, including problem-solving and learning.3 It can be used for a range of purposes, including automating tasks, identifying patterns in data, and synthesizing multiple sources of information. In health care, AI technologies are already used in fields that rely on image analysis, such as radiology and ophthalmology, and in products that process and analyze data from wearable sensors to detect diseases or infer the onset of other health conditions.4\n\nAI programs can also predict patient outcomes based on data collected from electronic health records, such as determining which patients may be at higher risk for disease or estimating who should receive increased monitoring. One such model identifies patients in the emergency room who may be at increased risk for developing sepsis based on factors such as vital signs and test results from electronic health records.5 Another hospital system has developed a model that aims to better predict which discharged patients are likely to be readmitted following their release compared with other risk-assessment tools.6 Other health care systems will likely follow suit in developing their own models as the technology becomes more accessible and well-established, and as federal regulations implement efforts to facilitate data exchange between electronic health record systems and mobile applications, a process known as interoperability.7\n\nFinally, AI can also play a role in research, including pharmaceutical development, combing through large sets of clinical data to improve a drug\u2019s design, predict its efficacy, and discover novel ways to treat diseases.8 The COVID-19 pandemic might help drive advances in AI in the clinical context, as hospitals and researchers have deployed it to support research, predict patient outcomes, and diagnose the disease.9 Some examples of AI products developed for use against COVID-19:10\n\u2022 COViage, a software prediction system, assesses whether hospitalized COVID-19 patients are at high risk of needing intubation.11\n\u2022 CLEWICU System, prediction software that identifies which ICU COVID-19 patients are at risk for respiratory failure or low blood pressure.12\n\u2022 Mount Sinai Health System developed an AI model that analyzes computed tomography (CT) scans of the chest and patient data to rapidly detect COVID-19.13\n\u2022 Researchers at the University of Minnesota, along with Epic Systems and M Health Fairview, developed an AI tool that can evaluate chest X-rays to diagnose possible cases of COVID-19.14\n\nAI can be developed using a variety of techniques. In traditional, or rules-based, approaches, an AI program will follow human-prescribed instructions for how to process data and make decisions, such as being programmed to alert a physician each time a patient with high blood pressure should be prescribed medication.15 Rules-based approaches are usually grounded in established best practices, such as clinical practice guidelines or literature.16 On the other hand, machine learning (ML) algorithms\u2014also referred to as a data-based approach\u2014\u201clearn\u201d from numerous examples in a dataset without being explicitly programmed to reach a particular answer or conclusion.17 ML algorithms can learn to decipher patterns in patient data at scales larger than a human can analyze while also potentially uncovering previously unrecognized correlations.18 Algorithms may also work at a faster pace than a human. These capabilities could be especially useful in health care settings, which can provide continuous streams of data from sources, including patient medical records and clinical studies.19\n\nMost ML-driven applications use a supervised approach in which the data used to train and validate the algorithm is labeled in advance by humans; for example, a collection of chest X-rays taken of people who have lung cancer and those who do not, with the two groups identified for the AI software. The algorithm examines all examples within the training dataset to \u201clearn\u201d which features of a chest X-ray are most closely correlated with the diagnosis of lung cancer and uses that analysis to predict new cases. Developers then test the algorithm to see how generalizable it is; that is, how well it performs on a new dataset, in this case, a new set of chest X-rays. Further validation is required by the end user, such as the health care practice, to ensure that the algorithm is accurate in real-world settings. Unsupervised learning is also possible, in which an algorithm does not receive labeled data and instead infers underlying patterns within a dataset.20\n\nLike any digital health tool, AI models can be flawed, presenting risks to patient safety. These issues can stem from a variety of factors, including problems with the data used to develop the algorithm, the choices that developers make in building and training the model, and how the AI-enabled program is eventually deployed.\n\nAI programs should be built and trained appropriately\n\nAI algorithms need to be trained on large, diverse datasets to be generalizable across a variety of populations and to ensure that they are not biased in a way that affects their accuracy and reliability. These challenges can resemble those for other health care products. For example, if a drug is tested in a clinical trial population that is not sufficiently representative of the actual populations it will be used in, it will not work as well when implemented in real-world clinical settings. In AI, similarly, any model must be evaluated carefully to ensure that its performance can be applied across a diverse set of patients and settings.\n\nHowever, such datasets are often difficult and expensive to assemble because of the fragmented U.S. health care system, characterized by multiple payers and unconnected health record systems. These factors can increase the propensity for error due to datasets that are incomplete or inappropriately merged from multiple sources.21 A 2020 analysis of data used to train image-based diagnostic AI systems found that approximately 70% of the studies that were included used data from three states, and that 34 states were not represented at all. Algorithms developed without considering geographic diversity, including variables such as disease prevalence and socioeconomic differences, may not perform as well as they should across a varied array of real-world settings.22\n\nThe data collection challenges and the inequalities embedded within the health care system contribute to bias in AI programs that can affect product safety and effectiveness and reinforce the disparities that have led to improper or insufficient treatment for many populations, particularly minority groups.23 For example, cardiovascular disease risks in populations of races and ethnicities that are not White have been both overestimated and underestimated by algorithms trained with data from the Framingham Heart Study, which mostly involved White patients.24 Similarly, if an algorithm developed to help detect melanoma is trained heavily on images of patients with lighter skin tones, it may not perform as well when analyzing lesions on people of color, who already present with more advanced skin disease and face lower survival rates than White patients.25\n\nBias can also occur when an algorithm developed in one setting, such as a large academic medical center, is applied in another, such as a small rural hospital with fewer resources. If not adapted and validated for its new context, an AI program may recommend treatments that are not available or appropriate in a facility with less access to specialists and cutting-edge technology.26\n\nMoreover, assembling sufficiently large patient datasets for AI-enabled programs can raise complex questions about data privacy and the ownership of personal health data. Protections to ensure that sensitive patient data remains anonymous are vital.27 Some health systems are sharing their patients\u2019 data with technology companies and digital startups to develop their AI-based programs, sometimes without those patients\u2019 knowledge. There is ongoing debate over whether patients should consent to having their data shared, and whether they should share in the profits from products that outside entities develop using their data.28 However, anonymizing patient data can pose its own challenges, as it can sometimes undermine efforts to ensure the representativeness of large datasets; if patient demographics are unknown to AI developers, then they may not be able to detect bias in the data.\n\nEnsuring safe and effective use of AI-enabled products\n\nOnce an AI-enabled program has been developed, it must be used in a way that ensures that it consistently performs as expected. This can be a complex undertaking, depending on the purpose of the AI model and how it is updated.\n\nML algorithms, for example, fall along a spectrum from \u201clocked\u201d to \u201cadaptive\u201d (also referred to as \u201ccontinuous learning\u201d). In a locked algorithm, the same input will always produce the same result unless the developer updates the program. In contrast, an adaptive algorithm has the potential to update itself based on new data, meaning that the same input could generate different decisions and recommendations over time.29 Either type of algorithm presents its own challenges.\n\nLocked algorithms can degrade as new treatments and clinical practices arise or as populations alter over time. These inevitable changes may make the real-world data entered into the AI program vastly different from its training data, leading the software to yield less accurate results. An adaptive algorithm could present an advantage in such situations, because it may learn to calibrate its recommendations in response to new data, potentially becoming more accurate than a locked model. However, allowing an adaptive algorithm to learn and adapt on its own also presents risks, including that it may infer patterns from biased practices or underperform in small subgroups of patients.30\n\nAI-enabled programs can also pose risks if they are not deployed appropriately and monitored carefully. One study found that a widely used algorithm disproportionately recommended White patients for high-risk care management programs, which provide intensive\u2014and often expensive\u2014services to people with complex health needs. Several health systems relied on the algorithm to identify patients who were most likely to benefit. However, the algorithm used higher health care costs as a proxy for medical need. Because Black patients are less likely to have access to care, even if they are insured, their health care costs tend to be lower. As a result, the algorithm systematically underestimated their health needs and excluded them from high-risk care programs.31\n\nOther challenges relate to the explainability of the output\u2014that is, how easy it is to explain to the end user how a program produced a certain result\u2014and the lack of transparency around how an AI-enabled program was developed. Some AI programs, for example, are referred to as \u201cblack-box\u201d models because the algorithms are derived from large datasets using complex techniques and reflect underlying patterns that may be too convoluted for a person, including the initial programmer, to understand. AI companies may also choose to keep their algorithms confidential, as proprietary information.32 Moreover, companies do not always publicly report detailed information on the datasets they use to develop or validate algorithms, limiting the ability of health care providers to evaluate how well the AI will perform for their patients. For example, a report examining companies\u2019 public summaries about their FDA-approved AI tools found that, of the 10 products approved for breast imaging, only one included a breakdown of the racial demographics in the dataset used to validate the algorithm. Breast cancer is significantly more likely to be fatal in Black women, who may be diagnosed at later stages of the disease and who experience greater barriers to care. Some or all of the AI devices in question may have been trained and validated on diverse patient populations, but the lack of public disclosure means that health care providers and patients might not have all the information they need to make informed decisions about the use of these products.33\n\nIn addition, patients are often not aware when an AI program has influenced the course of their care; these tools could, for example, be part of the reason a patient does not receive a certain treatment or is recommended for a potentially unnecessary procedure.34 Although there are many aspects of health care that a patient may not fully understand, in a recent patient engagement meeting hosted by FDA, some committee members\u2014including patient advocates\u2014expressed a desire to be notified when an AI product is part of their care. This desire included knowing if the data the model was trained on was representative of their particular demographics, or if it had been modified in some way that changed its intended use.35\n\nGiven the complexity of these products and the challenge of deploying them, health systems may need to recruit or train staff members with the technical skills to evaluate these models, understand their limitations, and implement them effectively. A provider\u2019s trust in\u2014and ability to correctly and appropriately use\u2014an AI tool is fundamental to its safety and effectiveness, and these human factors may vary significantly across institutions and even individuals.36 If providers do not understand how and why an algorithm arrived at a particular decision or result, they may struggle to interpret the result or apply it to a patient.\n\nSoftware developers, health care providers, policymakers, and patients all have a role to play in addressing these various challenges. Regulatory agencies also may need to adapt their current oversight processes to keep pace with the rapid shifts underway in this field.\n\nHow and under what circumstances does FDA regulate AI products?\n\nFDA is tasked with ensuring the safety and effectiveness of many AI-driven medical products. The agency largely regulates software based on its intended use and the level of risk to patients if it is inaccurate. If the software is intended to treat, diagnose, cure, mitigate, or prevent disease or other conditions, FDA considers it a medical device.37 Most products considered medical devices and that rely on AI/ML are categorized as Software as a Medical Device (SaMD).38 Examples of SaMD include software that helps detect and diagnose a stroke by analyzing MRI images, or computer-aided detection (CAD) software that processes images to aid in detecting breast cancer.39 Some consumer-facing products\u2014such as certain applications that run on a smartphone\u2014may also be classified as SaMD.40 By contrast, FDA refers to a computer program that is integral to the hardware of a medical device\u2014such as one that controls an X-ray panel\u2014as Software in a Medical Device.41 These products can also incorporate AI technologies.\n\nAs with any medical device, AI-enabled software is subject to FDA review based on its risk classification. Class I devices\u2014such as software that solely displays readings from a continuous glucose monitor\u2014 pose the lowest risk. Class II devices are considered to be moderate to high risk, and may include AI software tools that analyze medical images such as mammograms and flag suspicious findings for a radiologist to review.48 Most Class II devices undergo what is known as a 510(k) review (named for the relevant section of the Federal Food, Drug, and Cosmetic Act), in which a manufacturer demonstrates that its device is \u201csubstantially equivalent\u201d to an existing device on the market with the same intended use and technological characteristics.49 One study found that the majority of FDA-reviewed AI-based devices on the market have come through FDA\u2019s 510(k) pathway. However, the authors note that they relied on publicly available information, and because the agency does not require companies to categorize their devices as AI/ML-based in public documents, it is difficult to know the true number.50\n\nAlternatively, certain Class I and Class II device manufacturers may submit a De Novo request to FDA, which can be used for devices that are novel but whose safety and underlying technology are well understood, and which are therefore considered to be lower risk.51 Several AI-driven devices currently on the market\u2014such as IDx-DR, OsteoDetect, and ContaCT (see the text box, \u201cExamples of FDA Cleared or Approved AI-Enabled Products\u201d)\u2014are Class II devices that were reviewed through the De Novo pathway.52\n\nClass III devices pose the highest risk. They include products that are life-supporting, life-sustaining, or substantially important in preventing impairment of human health. These devices must undergo the full premarket approval process, and developers must submit clinical evidence that the benefits of the product outweigh the risks.53 The continuous glucose monitoring system, Guardian Connect system, was approved through a premarket approval.54\n\nOnce a device is on the market, FDA takes a risk-based approach to determine whether it will require premarket review of any changes the developer makes. In general, each time a manufacturer significantly updates the software or makes other changes that would substantially affect the device\u2019s performance, the device may be subject to additional review by FDA, although the process for this evaluation differs depending on the device\u2019s risk classification and the nature of the change.\n\nClinical decision support (CDS) software is a broad term that FDA defines as technologies that provide health care providers and patients with \u201cknowledge and person-specific information, intelligently filtered or presented at appropriate times to enhance health and health care.\u201d56 Studies have shown that CDS software can improve patient care.57 These products can have device and nondevice applications. To be exempt from the definition of device, and not regulated by the FDA, CDS software must meet criteria that Congress set in the 21st Century Cures Act of 2016. (See the text box, \u201cExemptions From FDA Review.\u201d)\n\nCrucially, the CDS software must support or provide recommendations to health care providers as they make clinical decisions, but the software cannot be intended toreplace a provider\u2019s independent judgment. That is, the software can inform decisions, but it cannot be intended as the driving factor behind them. Otherwise, the software must be regulated as a medical device by the agency. The distinction between informing and driving a decision can be difficult to assess and has proved challenging for FDA to describe as it attempts to implement the law. The agency released draft guidance in 2017 on how it would interpret those provisions with respect to CDS software. In response to feedback from product developers, which raised concerns that the agency was interpreting its authority too broadly, FDA officials revised and re-released the draft guidance in 2019. However, the 2019 guidance\u2014in which FDA attempted to harmonize its interpretation of the 21st Century Cures Act with existing international criteria for software\u2014has also drawn concerns from some health care provider organizations. They argue that the guidance may exclude too many types of software from review and that FDA needs to clarify how the agency would apply it to specific products.58\n\nThis is particularly the case for CDS products\u2014including those that rely on AI\u2014developed and used by health care providers. Some health systems may be developing or piloting AI-driven CDS software for use within their own facility that might technically meet the definition of a medical device. The distinction between the practice of medicine\u2014which FDA does not regulate\u2014and a device is unclear in circumstances in which a software program is developed and implemented within a single health care system and is not sold to an outside party. The agency has not publicly stated its position on this issue; however, current regulations do exempt licensed practitioners who manufacture or alter devices solely for use in their practice from product registration requirements.59\n\nHospital accrediting bodies (such as the Joint Commission), standards-setting organizations (such as the Association for the Advancement of Medical Instrumentation), and government actors may need to fill this gap in oversight to ensure patient safety as these tools are more widely adopted.60 For example, the Federal Trade Commission (FTC), which is responsible for protecting consumers and promoting fair market competition, published guidance in April 2020 for organizations using AI-enabled algorithms. Because algorithms that automate decision-making have the potential to produce negative or adverse outcomes for consumers, the guidance emphasizes the importance of using tools that are transparent, fair, robust, and explainable to the end consumer.61 One year later, the FTC announced that it may take action against those organizations whose algorithms may be biased or inaccurate.62\n\nFDA officials have acknowledged that the rapid pace of innovation in the digital health field poses a significant challenge for the agency. They say new regulatory frameworks will be essential to allow the agency to ensure the safety and effectiveness of the devices on the market without unnecessarily slowing progress.63\n\nIn 2019, the agency began piloting an oversight framework called the Software Precertification Program, which, if fully implemented, would be a significant departure from its normal review process. Rather than reviewing devices individually, FDA would first evaluate the developer. If the organization meets certain qualifications and demonstrates it has rigorous processes to develop safe, effective devices, it would be able to undergo a significantly streamlined review process and make changes or even introduce products without going through premarket review. Nine companies participated in this pilot program. The lessons learned may help inform the development of a future regulatory model for software-based medical devices.64 However, some members of Congress have questioned FDA\u2019s statutory authority to establish this program.65 Legislation may be required before FDA can fully implement it, and its appeal to software developers is not yet clear.\n\nThe agency has also proposed a regulatory framework targeted to SaMD products that rely on an adaptive learning approach. Thus far, FDA has only cleared or approved AI devices that rely on a \u201clocked\u201d algorithm, which does not change over time unless it is updated by the developer. Adaptive algorithms, by contrast, have the potential to incorporate new data and \u201clearn\u201d in real time, meaning that the level of risk or performance of the product may also change rapidly. Given the speed and sometimes unpredictable nature of these changes, it can be difficult to determine when the SaMD\u2019s algorithm may require additional review by the agency to ensure that it is still safe and effective for its intended use.\n\nIn a 2019 white paper, FDA outlined a potential approach to addressing this question of adaptive learning. It is based on four general principles:66\n\u2022 Clear expectations on quality systems and good machine learning practices. As with any device manufacturer, FDA expects SaMD developers to have an established system to ensure that their device meets the relevant quality standards and conforms to regulations. In addition, a developer would need to implement established best practices for developing an algorithm, known as Good Machine Learning Practices (GMLP). This set of standards is still evolving, and may eventually need to be included as an amendment to current Good Manufacturing Practice requirements for devices.67 FDA has recently stated it needs industry and stakeholder input to address outstanding questions on what these good practices look like for algorithm design, training, and testing.68\n\u2022 Premarket assessment of SaMD products that require it. Under this framework, developers would have the option to submit a plan for future modifications, called a predetermined change control plan, as part of the initial premarket review of an SaMD that relies on AI/ML. This plan would include the types of anticipated modifications that may occur and the approach the developer would use to implement those changes and reduce the associated risks.\n\u2022 Routine monitoring of SaMD products by manufacturers to determine when an algorithm change requires FDA review. Under the current regulatory framework, many changes to an SaMD product would likely require the developer to file a new premarket submission. In the proposed approach, if modifications are made within the bounds of the predetermined change control plan, developers would need only to document those changes. If the changes are beyond the scope of the change control plan but do not lead to a new intended use of the device (for example, the developer makes the SaMD compatible with other sources of data, or incorporates a different type of data), then FDA may perform a review of the change control plan alone and approve a new version. However, if the modifications lead to a new intended use (for example, by expanding the target patient population from adults to children), then FDA would likely need to conduct an additional premarket review.\n\u2022 Transparency and real-world performance monitoring. As part of this approach, FDA would expect a commitment from developers to adhere to certain principles of transparency and engage in ongoing performance monitoring. As such, developers would be expected to provide periodic reporting to the agency on implemented updates and performance metrics, among other requirements.\n\nThe proposed framework would be a significant shift in how FDA currently regulates devices, and\u2014as with the precertification program\u2014the agency has acknowledged that certain aspects of the framework may require congressional approval to implement.69 Even if permission is granted, there are outstanding questions about how this framework would be implemented in practice and applied to specific devices. FDA is currently working on a series of follow-up documents that will provide further details on its proposed approach.70\n\nMost recently, the agency published the \u201cArtificial Intelligence/Machine Learning (AI/ML)-Based Software as a Medical Device (SaMD) Action Plan,\u201d outlining its intended next steps. These include updating its proposed framework and issuing draft guidance on the predetermined change control plan, encouraging harmonization among technology developers on the development of GMLP, and holding a public workshop on medical device labeling to support transparency to end users. In addition, the agency will support efforts to develop methods for the evaluation and improvement of ML algorithms, including how to identify and eliminate bias, and to work with stakeholders to advance real-world performance monitoring pilots.71\n\nEspecially as the use of AI products in health care proliferates, FDA and other stakeholders will need to develop clear guidelines on the clinical evidence necessary to demonstrate the safety and effectiveness of such products and the extent to which product labels need to specify limitations on their performance and generalizability. As part of this effort, the agency could consider requiring developers to provide public information about the data used to validate and test AI devices so that end users can better understand their benefits and risks.\n\nFDA\u2019s recent SaMD Action Plan is a good step forward, but the agency will still need to clarify other key issues, including:\n\u2022 When a modification to SaMD or an adaptive ML device requires premarket review. The draft guidance on the predetermined change control plan could be a critical part of this policy.\n\u2022 Whether and how the Software Precertification Program can be extended beyond the pilot phase.\n\u2022 The distinction between software regulated by FDA and exempt software, which will turn heavily on the difference between informing clinical decisions and driving them.\n\u2022 How GMLP, when they are developed, will intersect with the current quality system regulations that apply to all devices.\n\u2022 How software updates and potential impacts on performance will be communicated to end users.\n\nIn addition, because there are products otherwise excluded from the definition of a medical device, another oversight body may need to play a role in ensuring patient safety, particularly for AI-enabled software not subject to FDA\u2019s authority. Further, for AI products used in the drug development process, FDA may need to provide additional guidance on the extent and type of evidence necessary to validate that products are working as intended.72\n\nTo fully seize the potential benefits that AI can add to the health care field while simultaneously ensuring the safety of patients, FDA may need to forge partnerships with a variety of stakeholders, including hospital accreditors, private technology firms, and other government actors such as the Office of the National Coordinator for Health Information Technology, which promulgates key standards for many software products, or the Centers for Medicare and Medicaid Services, which makes determinations about which technologies those insurance programs will cover. And, as previously mentioned, Congress may need to grant FDA additional authorities before the agency can implement some of its proposed policies, particularly as they relate to the precertification pilot.\n\nAI represents a transformational opportunity to improve patient outcomes, drive efficiency, and expedite research across health care. As such, health care providers, software developers, and researchers will continue to innovate and develop new AI products that test the current regulatory framework. FDA is attempting to meet these challenges and develop policies that can enable innovation while protecting public health, but there are many questions that the agency will need to address in order to ensure that this happens. As these policies evolve, legislative action may also be necessary to resolve the regulatory uncertainties within the sector.\n\nExplainability: The ability fordevelopers to explain in plain language how their data will be used.73\n\nGeneralizability: The accuracy with which results or findings can be transferred to other situations or people outside of those originally studied.74\n\nGood Machine Learning Practices (GMLP): AI/ML best practices (such as those for data management or evaluation), analogous to good software engineering practices or quality system practices.75\n\nMachine learning (ML): An AI technique that can be used to design and train software algorithms to learn from and act on data. These algorithms can be \u201clocked,\u201d so that their function does not change, or \u201cadaptive,\u201d meaning that their behavior can change over time.76\n\nSoftware as a Medical Device (SaMD): Defined by the International Medical Device Regulators Forum as \u201csoftware intended to be used for one or more medical purposes that perform these purposes without being part of a hardware medical device.\u201d77\n\u2022 E.J. Topol, \u201cHigh-Performance Medicine: The Convergence of Human and Artificial Intelligence,\u201d Nature Medicine 25 (2019): 44\u201356, https://www.nature.com/articles/s41591-018-0300-7.\n\u2022 U.S. Food and Drug Administration, \u201cArtificial Intelligence/Machine Learning (AI/ML)-Based Software as a Medical Device (SaMD) Action Plan\u201d (2021), https://www.fda.gov/media/145022/download; U.S. Food and Drug Administration, \u201cProposed Regulatory Framework for Modifications to Artificial Intelligence/Machine Learning (AI/ML)-Based Software as a Medical Device (SaMD)\u2014Discussion Paper and Request for Feedback\u201d (2019), https://www.fda.gov/media/122535/download.\n\u2022 G. Daniel et al., \u201cCurrent State and Near-Term Priorities for AI-Enabled Diagnostic Support Software in Health Care\u201d (Duke-Margolis Center for Health Policy, 2019), https://healthpolicy.duke.edu/sites/default/files/2019-11/dukemargolisaienableddxss.pdf.\n\u2022 K.-H. Yu, A.L. Beam, and I.S. Kohane, \u201cArtificial Intelligence in Healthcare,\u201d Nature Biomedical Engineering 2 (2018): 719\u201331, https://www.nature.com/articles/s41551-018-0305-z.\n\u2022 M. Garrity, \u201cU of Maryland Medical Systems Develops Machine Learning Model to Better Predict Readmissions,\u201d June 7, 2019, https://www.beckershospitalreview.com/artificial-intelligence/u-of-maryland-medical-systems-develops-machine-learning-model-to-better-predict-readmissions.html.\n\u2022 J. Vamathevan et al., \u201cApplications of Machine Learning in Drug Discovery and Development,\u201d Nature Reviews Drug Discovery 18 (2019): 463\u201377, https://www.nature.com/articles/s41573-019-0024-5.\n\u2022 L. Richardson, \u201cArtificial Intelligence Has Helped to Guide Pandemic Response, but Requires Adequate Regulation,\u201d The Pew Charitable Trusts, March 11, 2021, https://www.pewtrusts.org/en/research-and-analysis/articles/2021/03/11/artificial-intelligence-has-helped-to-guide-pandemic-response-but-requires-adequate-regulation; N. Arora, A.K. Banerjee, and M.L. Narasu, \u201cThe Role of Artificial Intelligence in Tackling COVID-19,\u201d Future Virology (2020), https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7692869/.\n\u2022 The first two products listed, COViage and the CLEWICU System, were granted Emergency Use Authorization (EUA) by FDA, which allows developers to market their products during a public health emergency without completing the agency\u2019s standard review process.\n\u2022 R. Robbins, \u201cFDA Issues Rare Emergency Authorization for an Algorithm Used to Inform COVID-19 Care,\u201d STAT+, Oct. 5, 2020, https://www.statnews.com/2020/10/05/dascena-algorithm-covid19-fda/; RADM Denise M. Hinton, chief scientist, U.S. Food and Drug Administration, letter to Ms. Carol Gu, vice president, Operations, Dascena Inc., \u201cEmergency Use Authorization (EUA) for Emergency Use of the COViage Hemodynamic Instability and Respiratory Decompensation Prediction System (COViage),\u201d Sept. 24, 2020, https://www.fda.gov/media/142454/download; Dascena Inc., \u201cDascena Receives FDA EUA for COVID-19 Hemodynamic Instability and Respiratory Decompensation Prediction System,\u201d Oct. 2, 2020, https://www.dascena.com/press-releases-old-dev/dascena-receives-fda-eua-for-covid-19-hemodynamic-instability-and-respiratory-decompensation-prediction-system.\n\u2022 N.P. Taylor, \u201cEmergency Authorization Granted to COVID-19 ICU Prediction Software,\u201d MedTech Dive, May 28, 2020, https://www.medtechdive.com/news/emergency-authorization-granted-to-covid-19-icu-prediction-software/578738/; RADM Denise M. Hinton, chief scientist, U.S. Food and Drug Administration, letter to CLEW Medical Ltd., c/o Ms. Yarmela Pavlovic, Manatt, Phelps & Phillips LLP, \u201cEmergency Use Authorization (EUA) for Emergency Use of the CLEWICU System,\u201d May 26, 2020, https://www.fda.gov/media/138369/download; CLEW, \u201cClew Receives FDA Emergency Use Authorization (EUA) for Its Predictive Analytics Platform in Support of COVID-19 Patients,\u201d June 16, 2020, https://clewmed.com/clew-receives-fda-emergency-use-authorization-eua-for-its-predictive-analytics-platform-in-support-of-covid-19-patients/.\n\u2022 Mount Sinai, \u201cMount Sinai First in U.S. To Use Artificial Intelligence to Analyze Coronavirus (COVID-19) Patients,\u201d May 19, 2020, https://www.mountsinai.org/about/newsroom/2020/mount-sinai-first-in-us-to-use-artificial-intelligence-to-analyze-coronavirus-covid19-patients-pr.\n\u2022 Epic, \u201cUniversity of Minnesota Develops AI Algorithm to Analyze Chest X-Rays for COVID-19,\u201d Oct. 1, 2020, https://www.epic.com/epic/post/university-minnesota-develops-ai-algorithm-analyze-chest-x-rays-covid-19; University of Minnesota, \u201cUniversity of Minnesota Develops AI Algorithm to Analyze Chest X-Rays for COVID-19,\u201d Oct. 1, 2020, https://twin-cities.umn.edu/news-events/university-minnesota-develops-ai-algorithm-analyze-chest-x-rays-covid-19.\n\u2022 A. Rajkomar, J. Dean, and I. Kohane, \u201cMachine Learning in Medicine,\u201d The New England Journal of Medicine 380, no. 14 (2019), https://www.nejm.org/doi/full/10.1056/NEJMra1814259.\n\u2022 Daniel et al., \u201cCurrent State and Near-Term Priorities for AI-Enabled Diagnostic Support Software in Health Care.\u201d\n\u2022 Rajkomar, Dean, and Kohane, \u201cMachine Learning in Medicine\u201d; Yu, Beam, and Kohane, \u201cArtificial Intelligence in Healthcare.\u201d\n\u2022 U.S. Food and Drug Administration, \u201cArtificial Intelligence and Machine Learning in Software as a Medical Device,\u201d last modified Jan. 12, 2021, https://www.fda.gov/medical-devices/software-medical-device-samd/artificial-intelligence-and-machine-learning-software-medical-device.\n\u2022 U.S. Food and Drug Administration, \u201cExecutive Summary for the Patient Engagement Advisory Committee Meeting\u201d (2020), https://www.fda.gov/media/142998/download; M. Matheny et al., \u201cArtificial Intelligence in Health Care: The Hope, the Hype, the Promise, the Peril\u201d (National Academy of Medicine, 2020), https://nam.edu/wp-content/uploads/2019/12/AI-in-Health-Care-PREPUB-FINAL.pdf; Yu, Beam, and Kohane, \u201cArtificial Intelligence in Healthcare.\u201d\n\u2022 W. Nicholson Price II, \u201cRisks and Remedies for Artificial Intelligence in Health Care,\u201d The Brookings Institution, Nov. 14, 2019, https://www.brookings.edu/research/risks-and-remedies-for-artificial-intelligence-in-health-care/.\n\u2022 A. Kaushal, R. Altman, and C. Langlotz, \u201cGeographic Distribution of U.S. Cohorts Used to Train Deep Learning Algorithms,\u201d Journal of the American Medical Association 324, no. 12 (2020), https://jamanetwork.com/journals/jama/article-abstract/2770833.\n\u2022 Nicholson Price II, \u201cRisks and Remedies for Artificial Intelligence in Health Care.\u201d\n\u2022 D.S. Char, N.H. Shah, and D. Magnus, \u201cImplementing Machine Learning in Health Care\u2014Addressing Ethical Challenges,\u201d New England Journal of Medicine 378, no. 11 (2018): 981\u201383, https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5962261/.\n\u2022 A.S. Adamson and A. Smith, \u201cMachine Learning and Health Care Disparities in Dermatology,\u201d JAMA Dermatology 154, no. 11 (2018): 1247-48, https://jamanetwork.com/journals/jamadermatology/article-abstract/2688587.\n\u2022 W.N.P. II, \u201cMedical AI and Contextual Bias,\u201d University of Michigan Public Law Research Paper No. 632; Harvard Journal of Law & Technology 33, no. 66 (2019), https://papers.ssrn.com/sol3/papers.cfm?abstract_id=3347890#.\n\u2022 C. Ross, \u201cAt Mayo Clinic, Sharing Patient Data With Companies Fuels AI Innovation\u2014and Concerns About Consent,\u201d STAT+, June 3, 2020, https://www.statnews.com/2020/06/03/mayo-clinic-patient-data-fuels-artificial-intelligence-consent-concerns/; S. Gerke, T. Minssen, and G. Cohen, \u201cEthical and Legal Challenges of Artificial Intelligence-Driven Healthcare,\u201d Artificial Intelligence in Healthcare (2020): 295\u2013336, https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7332220/.\n\u2022 U.S. Food and Drug Administration, \u201cProposed Regulatory Framework for Modifications to Artificial Intelligence/Machine Learning (AI/ML)-Based Software as a Medical Device (SaMD)\u201d (2021), https://www.fda.gov/files/medical devices/published/US-FDA-Artificial-Intelligence-and-Machine-Learning-Discussion-Paper.pdf; Daniel et al., \u201cCurrent State and Near-Term Priorities for AI-Enabled Diagnostic Support Software in Health Care.\u201d\n\u2022 Daniel et al., \u201cCurrent State and Near-Term Priorities for AI-Enabled Diagnostic Support Software in Health Care.\u201d\n\u2022 Z. Obermeyer et al., \u201cDissecting Racial Bias in an Algorithm Used to Manage the Health of Populations,\u201d Science 366, no. 6464 (2019): 447-53, https://science.sciencemag.org/content/366/6464/447; C.K. Johnson, \u201cRacial Bias in Health Care Software Aids Whites Over Blacks,\u201d The Seattle Times, Oct. 25, 2019, https://www.seattletimes.com/seattle-news/health/racial-bias-in-health-care-software-aids-whites-over-blacks/.\n\u2022 S. Samuel, \u201cAI Can Now Outperform Doctors at Detecting Breast Cancer. Here\u2019s Why It Won\u2019t Replace Them,\u201d Vox, Jan. 3, 2020, https://www.vox.com/future-perfect/2020/1/3/21046574/ai-google-breast-cancer-mammogram-deepmind; W.N.P. II, \u201cRegulating Black-Box Medicine,\u201d 116 Michigan Law Review 42 (2017), https://papers.ssrn.com/sol3/papers.cfm?abstract_id=2938391.\n\u2022 C. Ross, \u201cCould AI Tools for Breast Cancer Worsen Disparities? Patchy Public Data in FDA Filings Fuel Concern,\u201d STAT+, Feb. 11, 2021, https://www.statnews.com/2021/02/11/breast-cancer-disparities-artificial-intelligence-fda/; Centers for Disease Control and Prevention, \u201cPatterns and Trends in Age-Specific Black-White Differences in Breast Cancer Incidence and Mortality\u2014United States, 1999\u20132014,\u201d Morbidity and Mortality Weekly Report 65, no. 40 (2016): 1093\u201398, https://www.cdc.gov/mmwr/volumes/65/wr/mm6540a1.htm?CDC_AA_refVal=https%3A%2F%2Fwww.cdc.gov%2Fcancer%2Fdcpc%2Fresearch%2Farticles%2Fbreast_cancer_rates_women.htm.\n\u2022 R. Robbins and E. Brodwin, \u201cAn Invisible Hand: Patients Aren\u2019t Being Told About the AI Systems Advising Their Care,\u201d July 15, 2020, https://www.statnews.com/2020/07/15/artificial-intelligence-patient-consent-hospitals/.\n\u2022 C. Ross, \u201cBias, Consent, and Data Transparency: What Patients Want the FDA to Consider About AI in Medicine,\u201d STAT+, Oct. 26, 2020, https://www.statnews.com/2020/10/26/artificial-intelligence-bias-fda-patients/.\n\u2022 E. Brodwin, \u201c\u2018It\u2019s Really on Them to Learn\u2019: How the Rapid Rollout of AI Tools Has Fueled Frustration Among Clinicians,\u201d Dec. 17, 2020, https://www.statnews.com/2020/12/17/artificial-intelligence-doctors-nurses-frustrated-ai-hospitals/.\n\u2022 U.S. Food and Drug Administration, \u201cProposed Regulatory Framework for Modifications to Artificial Intelligence/Machine Learning (AI/ML)-Based Software as a Medical Device (SaMD).\u201d\n\u2022 U.S. Food and Drug Administration, \u201cExecutive Summary for the Patient Engagement Advisory Committee Meeting.\u201d\n\u2022 U.S. Food and Drug Administration, \u201cWhat Are Examples of Software as a Medical Device?\u201d last modified Dec. 6, 2017, https://www.fda.gov/medical-devices/software-medical-device-samd/what-are-examples-software-medical-device; Deloitte, \u201cSoftware as a Medical Device,\u201d accessed April 5, 2021, https://www2.deloitte.com/us/en/pages/public-sector/articles/software-as-a-medical-device-fda.html.\n\u2022 T. Minssen et al., \u201cRegulatory Responses to Medical Machine Learning,\u201d Journal of Law and the Biosciences (2020), https://academic.oup.com/jlb/advance-article/doi/10.1093/jlb/lsaa002/5817484.\n\u2022 U.S. Food and Drug Administration, \u201cSoftware as a Medical Device (SaMD),\u201d last modified Dec. 4, 2018, https://www.fda.gov/medical-devices/digital-health-center-excellence/software-medical-device-samd; Deloitte, \u201cSoftware as a Medical Device.\u201d\n\u2022 U.S. Food and Drug Administration, \u201cFDA Permits Marketing of Artificial Intelligence-Based Device to Detect Certain Diabetes-Related Eye Problems,\u201d April 11, 2018, https://www.fda.gov/news-events/press-announcements/fda-permits-marketing-artificial-intelligence-based-device-detect-certain-diabetes-related-eye; Digital Diagnostics, \u201cIDx-DR,\u201d accessed April 5, 2021, https://dxs.ai/products/idx-dr/idx-dr-overview/.\n\u2022 U.S. Food and Drug Administration, \u201cFDA Permits Marketing of Artificial Intelligence Algorithm for Aiding Providers in Detecting Wrist Fractures,\u201d May 24, 2018, https://www.fda.gov/news-events/press-announcements/fda-permits-marketing-artificial-intelligence-algorithm-aiding-providers-detecting-wrist-fractures; U.S. Food and Drug Administration, \u201cEvaluation of Automatic Class III Designation for Osteodetect\u201d (https://www.accessdata.fda.gov/cdrh_docs/reviews/DEN180005.pdf.\n\u2022 U.S. Food and Drug Administration, \u201cFDA Permits Marketing of Clinical Decision Support Software for Alerting Providers of a Potential Stroke in Patients,\u201d Feb. 13, 2018, https://www.fda.gov/news-events/press-announcements/fda-permits-marketing-clinical-decision-support-software-alerting-providers-potential-stroke; U.S. Food and Drug Administration, \u201cEvaluation of Automatic Class III Designation for Contact,\u201d https://www.accessdata.fda.gov/cdrh_docs/reviews/DEN170073.pdf.\n\u2022 U.S. Food and Drug Administration, \u201cGuardian Connect System\u2014P160007,\u201d last modified April 8, 2018, https://www.fda.gov/medical-devices/recently-approved-devices/guardian-connect-system-p160007; The Medical Futurist, \u201cMedtronic\u2019s Smart Continuous Glucose Monitoring System Rolls Out This Summer,\u201d March 21, 2018, https://medicalfuturist.com/medtronics-smart-continuous-glucose-monitoring-system-rolls-summer/; Medtronic, \u201cThe Guardian\u2122 Connect System,\u201d accessed April 5, 2021, https://www.medtronicdiabetes.com/products/guardian-connect-continuous-glucose-monitoring-system.\n\u2022 Empatica, \u201cIndications for Use and Safety Information,\u201d accessed April 5, 2021, https://www.empatica.com/embrace-IFU; Empatica, \u201cHow Does Embrace2 Work? Watch Our Animated Video!\u201d June 7, 2019, https://www.empatica.com/blog/how-does-embrace2-work-watch-our-animated-video.html\n\u2022 Daniel et al., \u201cCurrent State and Near-Term Priorities for AI-Enabled Diagnostic Support Software in Health Care\u201d; for example: T. Mills, director, Division of Radiological Health, Office of In Vitro Diagnostics and Radiological Health, Center for Devices and Radiological Health, letter to Kevin Harris, CEO, CureMetrix Inc., \u201cReply to Section 510(k) Premarket Notification of Intent,\u201d March 8, 2019, http://www.accessdata.fda.gov/cdrh_docs/pdf18/K183285.pdf.\n\u2022 U.S. Food and Drug Administration, \u201cOverview of Device Regulation,\u201d last modified Sept. 4, 2020, https://www.fda.gov/medical-devices/device-advice-comprehensive-regulatory-assistance/overview-device-regulation; U.S. Food and Drug Administration, \u201cStep 3: Pathway to Approval,\u201d last modified Feb. 9, 2018, https://www.fda.gov/patients/device-development-process/step-3-pathway-approval.\n\u2022 S. Benjamens, P. Dhunnoo, and B. Mesk\u00f3, \u201cThe State of Artificial Intelligence-Based FDA-Approved Medical Devices and Algorithms: An Online Database,\u201d npj Digital Medicine, Vol.3, no. 118 (2018), https://www.nature.com/articles/s41746-020-00324-0.\n\u2022 U.S. Food and Drug Administration, \u201cStep 3: Pathway to Approval\u201d; Daniel et al., \u201cCurrent State and Near-Term Priorities for AI-Enabled Diagnostic Support Software in Health Care.\u201d\n\u2022 U.S. Food and Drug Administration, \u201cDe Novo Classification Request for IDx-DR\u201d (2018), https://www.accessdata.fda.gov/cdrh_docs/reviews/DEN180001.pdf; U.S. Food and Drug Administration, \u201cEvaluation of Automatic Class III Designation for Osteodetect\u201d; U.S. Food and Drug Administration, \u201cEvaluation of Automatic Class III Designation for Contact.\u201d\n\u2022 J. Jin, \u201cFDA Authorization of Medical Devices,\u201d JAMA 311, no. 4 (2014): 435, https://jamanetwork.com/journals/jama/fullarticle/1817798; U.S. Food and Drug Administration, \u201cStep 3: Pathway to Approval.\u201d\n\u2022 C.H. Lias, director, Division of Chemistry and Toxicology Devices, Office of In Vitro Diagnostics and Radiological Health, Center for Devices and Radiological Health, letter to Liane Miller, Medtronic MiniMed, Regulatory Affairs Manager, \u201cPremarket Approval Application (PMA) Review,\u201d https://www.accessdata.fda.gov/cdrh_docs/pdf16/P160007a.pdf.\n\u2022 Section 520(o) of the Federal Food, Drug, and Cosmetic Act; United States Public Law 114-255\u201421st Century Cures Act (2016), https://www.congress.gov/114/plaws/publ255/PLAW-114publ255.pdf; U.S. Food and Drug Administration, \u201cChanges to Existing Medical Software Policies Resulting from Section 3060 of the 21st Century Cures Act\u2014Guidance for Industry and Food and Drug Administration Staff\u201d (Sept. 27, 2019), https://www.fda.gov/media/109622/download.\n\u2022 U.S. Food and Drug Administration, \u201cClinical Decision Support Software\u2014Draft Guidance for Industry and Food and Drug Administration Staff\u201d (2019), https://www.fda.gov/media/109618/download.\n\u2022 For example: J.M. Sperl-Hillen et al., \u201cClinical Decision Support Directed to Primary Care Patients and Providers Reduces Cardiovascular Risk: A Randomized Trial,\u201d Journal of the American Medical Informatics Association 25, no. 9 (2018): 1137-46, https://pubmed.ncbi.nlm.nih.gov/29982627/.\n\u2022 Nicholson Price II, \u201cRisks and Remedies for Artificial Intelligence in Health Care.\u201d\n\u2022 E. Jillson, \u201cAiming for Truth, Fairness, and Equity in Your Company\u2019s Use of AI,\u201d Federal Trade Commission, April 19, 2021, https://www.ftc.gov/news-events/blogs/business-blog/2021/04/aiming-truth-fairness-equity-your-companys-use-ai.\n\u2022 U.S. Food and Drug Administration, \u201cStatement from FDA Commissioner Scott Gottlieb, M.D., and Center for Devices and Radiological Health Director Jeff Shuren, M.D., J.D., on Agency Efforts to Work with Tech Industry to Spur Innovation in Digital Health,\u201d Sept. 12, 2018, https://www.fda.gov/news-events/press-announcements/statement-fda-commissioner-scott-gottlieb-md-and-center-devices-and-radiological-health-director; U.S. Food and Drug Administration, \u201cStatement from FDA Commissioner Scott Gottlieb, M.D., On Steps Toward a New, Tailored Review Framework for Artificial Intelligence-Based Medical Devices,\u201d April 02, 2019, https://www.fda.gov/news-events/press-announcements/statement-fda-commissioner-scott-gottlieb-md-steps-toward-new-tailored-review-framework-artificial; U.S. Food and Drug Administration, \u201cFDA Launches the Digital Health Center of Excellence,\u201d Sept. 22, 2020, https://www.fda.gov/news-events/press-announcements/fda-launches-digital-health-center-excellence.\n\u2022 Senator Elizabeth Warren, Senator Patty Murray, and Senator Tina Smith, letter to Scott Gottlieb, commissioner, U.S. Food and Drug Administration, and Jeffrey Shuren, director, Center for Devices and Radiological Health, U.S. Food and Drug Administration, \u201cLetter to FDA on Regulation of Software as Medical Device,\u201d Oct. 10, 2018, https://www.warren.senate.gov/imo/media/doc/2018.10.10%20Letter%20to%20FDA%20on%20regulation%20of%20sofware%20as%20medical%20device.pdf.\n\u2022 U.S. Food and Drug Administration, \u201cArtificial Intelligence and Machine Learning in Software as a Medical Device\u201d; U.S. Food and Drug Administration, \u201cProposed Regulatory Framework for Modifications to Artificial Intelligence/Machine Learning (AI/ML)-Based Software as a Medical Device (SaMD).\u201d\n\u2022 U.S. Food and Drug Administration, \u201cProposed Regulatory Framework for Modifications to Artificial Intelligence/Machine Learning (AI/ML)-Based Software as a Medical Device (SaMD)\u2014Discussion Paper and Request for Feedback\u201d; U.S. Food and Drug Administration, \u201cDeveloping the Software Precertification Program: Summary of Learnings and Ongoing Activities\u201d (2020), https://www.fda.gov/media/142107/download.\n\u2022 U.S. Food and Drug Administration, \u201cFDA Releases Artificial Intelligence/Machine Learning Action Plan,\u201d Jan. 12, 2021, https://www.fda.gov/news-events/press-announcements/fda-releases-artificial-intelligencemachine-learning-action-plan; U.S. Food and Drug Administration, \u201cArtificial Intelligence/Machine Learning (AI/ML)-Based Software as a Medical Device (SaMD) Action Plan.\u201d\n\u2022 U.S. Government Accountability Office and National Academy of Medicine, \u201cArtificial Intelligence in Health Care Benefits and Challenges of Machine Learning in Drug Development\u201d (2019),www.gao.gov/assets/gao-20-215sp.pdf.\n\u2022 U.S. Food and Drug Administration, \u201cExecutive Summary for the Patient Engagement Advisory Committee Meeting.\u201d\n\u2022 U.S. Food and Drug Administration, \u201cProposed Regulatory Framework for Modifications to Artificial Intelligence/Machine Learning (AI/ML)-Based Software as a Medical Device (SaMD) \u2014Discussion Paper and Request for Feedback.\u201d\n\u2022 U.S. Food and Drug Administration, \u201cArtificial Intelligence and Machine Learning in Software as a Medical Device.\u201d\n\u2022 International Medical Device Regulators Forum, \u201cSoftware as a Medical Device (SaMD): Key Definitions\u201d (2013), http://www.imdrf.org/docs/imdrf/final/technical/imdrf-tech-131209-samd-key-definitions-140901.pdf; U.S. Food and Drug Administration, \u201cSoftware as a Medical Device (SaMD).\u201d", "cleaned_text_custom_parser": "Our projects are ambitious, consistent with a tested investment philosophy, and designed to reap measurable benefits for the public. Throughout our history, Pew has focused on impact\u2014asking tough questions, seeking out evidence, working with strong partners, bringing unique interests together, and striving for effective solutions that address important issues.\nOur affiliatePew Research Centeris focused around tracking important changes\u2014using data-based research\u2014to help policy analysts, government officials, and the public identify and prepare for future challenges.\nAs part of our mission to inform the public, Pew experts offer nonpartisan, rigorous reports, research, and recommendations. We also share research findings, analysis, essays, and other insights in our magazines and podcast.\nReceive our best conservation research bi-weekly\u2014stunning photos, wins, and action alerts.\nFor more than 75 years, we have used data to make a difference\u2014addressing the challenges of a changing world by illuminating issues, creating common ground, and advancing ambitious strategies that lead to tangible progress.\nHow FDA Regulates Artificial Intelligence in Medical Products\nOverview\nHealth care organizations are using artificial intelligence (AI)\u2014which the U.S. Food and Drug Administration defines as \u201cthe science and engineering of making intelligent machines\u201d\u2014for a growing range of clinical, administrative, and research purposes. This AI software can, for example, help health care providers diagnose diseases, monitor patients\u2019 health, or assist with rote functions such as scheduling patients.\nAlthough AI offers unique opportunities to improve health care and patient outcomes, it also comes with potential challenges. AI-enabled products, for example, have sometimes resulted in inaccurate, even potentially harmful, recommendations for treatment.1These errors can be caused by unanticipated sources of bias in the information used to build or train the AI, inappropriate weight given to certain data points analyzed by the tool, and other flaws.\nThe regulatory framework governing these tools is complex. FDA regulates some\u2014but not all\u2014AI-enabled products used in health care, and the agency plays an important role in ensuring the safety and effectiveness of those products under its jurisdiction. The agency is currently considering how to adapt its review process for AI-enabled medical devices that have the ability to evolve rapidly in response to new data, sometimes in ways that are difficult to foresee.2\nThis brief describes current and potential uses of AI in health care settings and the challenges these technologies pose, outlines how and under what circumstances they are regulated by FDA, and highlights key questions that will need to be addressed to ensure that the benefits of these devices outweigh their risks. It will take a collective effort by FDA, Congress, technology developers, and the health care industry to ensure the safety and effectiveness of AI-enabled technology.\nWhat is AI and how is it used in health care?\nAI refers to the ability of a machine to perform a task that mimics human behavior, including problem-solving and learning.3It can be used for a range of purposes, including automating tasks, identifying patterns in data, and synthesizing multiple sources of information. In health care, AI technologies are already used in fields that rely on image analysis, such as radiology and ophthalmology, and in products that process and analyze data from wearable sensors to detect diseases or infer the onset of other health conditions.4\nAI programs can also predict patient outcomes based on data collected from electronic health records, such as determining which patients may be at higher risk for disease or estimating who should receive increased monitoring. One such model identifies patients in the emergency room who may be at increased risk for developing sepsis based on factors such as vital signs and test results from electronic health records.5Another hospital system has developed a model that aims to better predict which discharged patients are likely to be readmitted following their release compared with other risk-assessment tools.6Other health care systems will likely follow suit in developing their own models as the technology becomes more accessible and well-established, and as federal regulations implement efforts to facilitate data exchange between electronic health record systems and mobile applications, a process known as interoperability.7\nFinally, AI can also play a role in research, including pharmaceutical development, combing through large sets of clinical data to improve a drug\u2019s design, predict its efficacy, and discover novel ways to treat diseases.8The COVID-19 pandemic might help drive advances in AI in the clinical context, as hospitals and researchers have deployed it to support research, predict patient outcomes, and diagnose the disease.9Some examples of AI products developed for use against COVID-19:10\nCOViage, a software prediction system, assesses whether hospitalized COVID-19 patients are at high risk of needing intubation.11\nCLEWICU System, prediction software that identifies which ICU COVID-19 patients are at risk for respiratory failure or low blood pressure.12\nMount Sinai Health System developed an AI model that analyzes computed tomography (CT) scans of the chest and patient data to rapidly detect COVID-19.13\nResearchers at the University of Minnesota, along with Epic Systems and M Health Fairview, developed an AI tool that can evaluate chest X-rays to diagnose possible cases of COVID-19.14\nHow are AI products developed?\nAI can be developed using a variety of techniques. In traditional, or rules-based, approaches, an AI program will follow human-prescribed instructions for how to process data and make decisions, such as being programmed to alert a physician each time a patient with high blood pressure should be prescribed medication.15Rules-based approaches are usually grounded in established best practices, such as clinical practice guidelines or literature.16On the other hand, machine learning (ML) algorithms\u2014also referred to as a data-based approach\u2014\u201clearn\u201d from numerous examples in a dataset without being explicitly programmed to reach a particular answer or conclusion.17ML algorithms can learn to decipher patterns in patient data at scales larger than a human can analyze while also potentially uncovering previously unrecognized correlations.18Algorithms may also work at a faster pace than a human. These capabilities could be especially useful in health care settings, which can provide continuous streams of data from sources, including patient medical records and clinical studies.19\nMost ML-driven applications use a supervised approach in which the data used to train and validate the algorithm is labeled in advance by humans; for example, a collection of chest X-rays taken of people who have lung cancer and those who do not, with the two groups identified for the AI software. The algorithm examines all examples within the training dataset to \u201clearn\u201d which features of a chest X-ray are most closely correlated with the diagnosis of lung cancer and uses that analysis to predict new cases. Developers then test the algorithm to see how generalizable it is; that is, how well it performs on a new dataset, in this case, a new set of chest X-rays. Further validation is required by the end user, such as the health care practice, to ensure that the algorithm is accurate in real-world settings. Unsupervised learning is also possible, in which an algorithm does not receive labeled data and instead infers underlying patterns within a dataset.20\nChallenges and risks with AI-enabled products\nLike any digital health tool, AI models can be flawed, presenting risks to patient safety. These issues can stem from a variety of factors, including problems with the data used to develop the algorithm, the choices that developers make in building and training the model, and how the AI-enabled program is eventually deployed.\nAI programs should be built and trained appropriately\nAI algorithms need to be trained on large, diverse datasets to be generalizable across a variety of populations and to ensure that they are not biased in a way that affects their accuracy and reliability. These challenges can resemble those for other health care products. For example, if a drug is tested in a clinical trial population that is not sufficiently representative of the actual populations it will be used in, it will not work as well when implemented in real-world clinical settings. In AI, similarly, any model must be evaluated carefully to ensure that its performance can be applied across a diverse set of patients and settings.\nHowever, such datasets are often difficult and expensive to assemble because of the fragmented U.S. health care system, characterized by multiple payers and unconnected health record systems. These factors can increase the propensity for error due to datasets that are incomplete or inappropriately merged from multiple sources.21A 2020 analysis of data used to train image-based diagnostic AI systems found that approximately 70% of the studies that were included used data from three states, and that 34 states were not represented at all. Algorithms developed without considering geographic diversity, including variables such as disease prevalence and socioeconomic differences, may not perform as well as they should across a varied array of real-world settings.22\nThe data collection challenges and the inequalities embedded within the health care system contribute to bias in AI programs that can affect product safety and effectiveness and reinforce the disparities that have led to improper or insufficient treatment for many populations, particularly minority groups.23For example, cardiovascular disease risks in populations of races and ethnicities that are not White have been both overestimated and underestimated by algorithms trained with data from the Framingham Heart Study, which mostly involved White patients.24Similarly, if an algorithm developed to help detect melanoma is trained heavily on images of patients with lighter skin tones, it may not perform as well when analyzing lesions on people of color, who already present with more advanced skin disease and face lower survival rates than White patients.25\nBias can also occur when an algorithm developed in one setting, such as a large academic medical center, is applied in another, such as a small rural hospital with fewer resources. If not adapted and validated for its new context, an AI program may recommend treatments that are not available or appropriate in a facility with less access to specialists and cutting-edge technology.26\nMoreover, assembling sufficiently large patient datasets for AI-enabled programs can raise complex questions about data privacy and the ownership of personal health data. Protections to ensure that sensitive patient data remains anonymous are vital.27Some health systems are sharing their patients\u2019 data with technology companies and digital startups to develop their AI-based programs, sometimes without those patients\u2019 knowledge. There is ongoing debate over whether patients should consent to having their data shared, and whether they should share in the profits from products that outside entities develop using their data.28However, anonymizing patient data can pose its own challenges, as it can sometimes undermine efforts to ensure the representativeness of large datasets; if patient demographics are unknown to AI developers, then they may not be able to detect bias in the data.\nEnsuring safe and effective use of AI-enabled products\nOnce an AI-enabled program has been developed, it must be used in a way that ensures that it consistently performs as expected. This can be a complex undertaking, depending on the purpose of the AI model and how it is updated.\nML algorithms, for example, fall along a spectrum from \u201clocked\u201d to \u201cadaptive\u201d (also referred to as \u201ccontinuous learning\u201d). In a locked algorithm, the same input will always produce the same result unless the developer updates the program. In contrast, an adaptive algorithm has the potential to update itself based on new data, meaning that the same input could generate different decisions and recommendations over time.29Either type of algorithm presents its own challenges.\nLocked algorithms can degrade as new treatments and clinical practices arise or as populations alter over time. These inevitable changes may make the real-world data entered into the AI program vastly different from its training data, leading the software to yield less accurate results. An adaptive algorithm could present an advantage in such situations, because it may learn to calibrate its recommendations in response to new data, potentially becoming more accurate than a locked model. However, allowing an adaptive algorithm to learn and adapt on its own also presents risks, including that it may infer patterns from biased practices or underperform in small subgroups of patients.30\nAI-enabled programs can also pose risks if they are not deployed appropriately and monitored carefully. One study found that a widely used algorithm disproportionately recommended White patients for high-risk care management programs, which provide intensive\u2014and often expensive\u2014services to people with complex health needs. Several health systems relied on the algorithm to identify patients who were most likely to benefit. However, the algorithm used higher health care costs as a proxy for medical need. Because Black patients are less likely to have access to care, even if they are insured, their health care costs tend to be lower. As a result, the algorithm systematically underestimated their health needs and excluded them from high-risk care programs.31\nOther challenges relate to the explainability of the output\u2014that is, how easy it is to explain to the end user how a program produced a certain result\u2014and the lack of transparency around how an AI-enabled program was developed. Some AI programs, for example, are referred to as \u201cblack-box\u201d models because the algorithms are derived from large datasets using complex techniques and reflect underlying patterns that may be too convoluted for a person, including the initial programmer, to understand. AI companies may also choose to keep their algorithms confidential, as proprietary information.32Moreover, companies do not always publicly report detailed information on the datasets they use to develop or validate algorithms, limiting the ability of health care providers to evaluate how well the AI will perform for their patients. For example, a report examining companies\u2019 public summaries about their FDA-approved AI tools found that, of the 10 products approved for breast imaging, only one included a breakdown of the racial demographics in the dataset used to validate the algorithm. Breast cancer is significantly more likely to be fatal in Black women, who may be diagnosed at later stages of the disease and who experience greater barriers to care. Some or all of the AI devices in question may have been trained and validated on diverse patient populations, but the lack of public disclosure means that health care providers and patients might not have all the information they need to make informed decisions about the use of these products.33\nIn addition, patients are often not aware when an AI program has influenced the course of their care; these tools could, for example, be part of the reason a patient does not receive a certain treatment or is recommended for a potentially unnecessary procedure.34Although there are many aspects of health care that a patient may not fully understand, in a recent patient engagement meeting hosted by FDA, some committee members\u2014including patient advocates\u2014expressed a desire to be notified when an AI product is part of their care. This desire included knowing if the data the model was trained on was representative of their particular demographics, or if it had been modified in some way that changed its intended use.35\nGiven the complexity of these products and the challenge of deploying them, health systems may need to recruit or train staff members with the technical skills to evaluate these models, understand their limitations, and implement them effectively. A provider\u2019s trust in\u2014and ability to correctly and appropriately use\u2014an AI tool is fundamental to its safety and effectiveness, and these human factors may vary significantly across institutions and even individuals.36If providers do not understand how and why an algorithm arrived at a particular decision or result, they may struggle to interpret the result or apply it to a patient.\nSoftware developers, health care providers, policymakers, and patients all have a role to play in addressing these various challenges. Regulatory agencies also may need to adapt their current oversight processes to keep pace with the rapid shifts underway in this field.\nHow and under what circumstances does FDA regulate AI products?\nFDA is tasked with ensuring the safety and effectiveness of many AI-driven medical products. The agency largely regulates software based on its intended use and the level of risk to patients if it is inaccurate. If the software is intended to treat, diagnose, cure, mitigate, or prevent disease or other conditions, FDA considers it a medical device.37Most products considered medical devices and that rely on AI/ML are categorized as Software as a Medical Device (SaMD).38Examples of SaMD include software that helps detect and diagnose a stroke by analyzing MRI images, or computer-aided detection (CAD) software that processes images to aid in detecting breast cancer.39Some consumer-facing products\u2014such as certain applications that run on a smartphone\u2014may also be classified as SaMD.40By contrast, FDA refers to a computer program that is integral to the hardware of a medical device\u2014such as one that controls an X-ray panel\u2014as Software in a Medical Device.41These products can also incorporate AI technologies.\nExamples of FDA Cleared or Approved AI-Enabled Products\nIDx-DR: Detects diabetic retinopathy\nThis software analyzes images of the eye to determine whether the patient should be\n referred to an eye professional because the images portray more than mild diabetic\n retinopathy or the patient should be rescreened in a year because the images were\n negative for more than mild diabetic retinopathy.42\nOsteoDetect: Detects and diagnoses wrist fractures\nThis software analyzes X-rays for signs of distal radius fracture and marks the location\n of the fracture to aid in detection and diagnosis.43\nContaCT: Detects a possible stroke and notifies a specialist\nThis software analyzes CT images of the brain for indicators usually associated with\n a stroke, and immediately texts a specialist if a suspected large vessel blockage is\n identified, potentially involving the specialist sooner than the usual standard of care.44\nGuardian Connect System: Continuous glucose monitoring system\nThis product monitors glucose levels in the tissues of a diabetic patient, using a sensor\n inserted under the skin, either on an arm or on the abdomen. A transmitter processes and\n sends this information wirelessly to an application installed on a mobile device. Patients\n can use the program to monitor whether their glucose levels are too low or high.45\nEmbrace2: Wearable seizure monitoring device\nThis product monitors physiological signals through a device worn on the wrist. If the\n technology senses activity that may indicate a seizure, it will send a command to a\n paired wireless device programmed to alert the patient\u2019s designated caregiver. The\n system will also record and store data from its sensors for future review by a health\n care professional.46\nFibriCheck: Mobile application to detect atrial fibrillation\nThis application uses either a smartphone camera or sensors in a smartwatch to\n analyze and record heart rhythms. This information, along with symptoms the patient\n is prompted to enter, are aggregated into a report along with next steps for the patient,\n if necessary.47\nAs with any medical device, AI-enabled software is subject to FDA review based on its risk classification. Class I devices\u2014such as software that solely displays readings from a continuous glucose monitor\u2014 pose the lowest risk. Class II devices are considered to be moderate to high risk, and may include AI software tools that analyze medical images such as mammograms and flag suspicious findings for a radiologist to review.48Most Class II devices undergo what is known as a 510(k) review (named for the relevant section of the Federal Food, Drug, and Cosmetic Act), in which a manufacturer demonstrates that its device is \u201csubstantially equivalent\u201d to an existing device on the market with the same intended use and technological characteristics.49One study found that the majority of FDA-reviewed AI-based devices on the market have come through FDA\u2019s 510(k) pathway. However, the authors note that they relied on publicly available information, and because the agency does not require companies to categorize their devices as AI/ML-based in public documents, it is difficult to know the true number.50\nAlternatively, certain Class I and Class II device manufacturers may submit a De Novo request to FDA, which can be used for devices that are novel but whose safety and underlying technology are well understood, and which are therefore considered to be lower risk.51Several AI-driven devices currently on the market\u2014such as IDx-DR, OsteoDetect, and ContaCT (see the text box, \u201cExamples of FDA Cleared or Approved AI-Enabled Products\u201d)\u2014are Class II devices that were reviewed through the De Novo pathway.52\nClass III devices pose the highest risk. They include products that are life-supporting, life-sustaining, or substantially important in preventing impairment of human health. These devices must undergo the full premarket approval process, and developers must submit clinical evidence that the benefits of the product outweigh the risks.53The continuous glucose monitoring system, Guardian Connect system, was approved through a premarket approval.54\nOnce a device is on the market, FDA takes a risk-based approach to determine whether it will require premarket review of any changes the developer makes. In general, each time a manufacturer significantly updates the software or makes other changes that would substantially affect the device\u2019s performance, the device may be subject to additional review by FDA, although the process for this evaluation differs depending on the device\u2019s risk classification and the nature of the change.\nExemptions From FDA Review\nCongress excluded certain health-related software from the definition of a medical device in the 21stCentury Cures Act of 2016.55Exemptions in the law include software that:\nExample: software for scheduling, practice and inventory management, or to process and maintain financial records.\nIs intended for maintaining or encouraging a healthy lifestyle and is unrelated to the diagnosis, cure, mitigation, prevention, or treatment of a disease or condition.\nExample: mobile applications that actively monitor exercise, provide daily motivational tips to reduce stress, or offer tools to promote or encourage healthy eating.\nIs intended to serve as electronic patient records, including patient-provided information, and its function is not intended to interpret or analyze patient records (including imaging data) for the purpose of diagnosis, cure, mitigation, prevention, or treatment.\nExamples: mobile applications that allow patients with a certain medical condition to record measurements or other events to share with their health care provider as part of a disease management plan, or that allow health care providers to access their patient\u2019s personal health record hosted on a web-based or other platform.\nIs intended for transferring, storing, converting formats, or displaying data or results.\nExample: software functions that display medical device data without modifying the data.\nIs not intended to acquire, process, or analyze data from scanning or diagnostic devices such as MRIs orin vitroclinical tests, AND is used for the purpose of:\nDisplaying, analyzing, or printing medical data, such as patient information or peer-reviewed clinical studies.\nSupporting or providing medical recommendations to a health care professional, on the condition that the software allows the provider to independently review how those recommendations were made.\nClinical decision support (CDS) software is a broad term that FDA defines as technologies that provide health care providers and patients with \u201cknowledge and person-specific information, intelligently filtered or presented at appropriate times to enhance health and health care.\u201d56Studies have shown that CDS software can improve patient care.57These products can have device and nondevice applications. To be exempt from the definition of device, and not regulated by the FDA, CDS software must meet criteria that Congress set in the 21stCentury Cures Act of 2016. (See the text box, \u201cExemptions From FDA Review.\u201d)\nCrucially, the CDS software must support or provide recommendations to health care providers as they make clinical decisions, but the software cannot be intended toreplace a provider\u2019s independent judgment. That is, the software can inform decisions, but it cannot be intended as the driving factor behind them. Otherwise, the software must be regulated as a medical device by the agency. The distinction betweeninforminganddrivinga decision can be difficult to assess and has proved challenging for FDA to describe as it attempts to implement the law. The agency released draft guidance in 2017 on how it would interpret those provisions with respect to CDS software. In response to feedback from product developers, which raised concerns that the agency was interpreting its authority too broadly, FDA officials revised and re-released the draft guidance in 2019. However, the 2019 guidance\u2014in which FDA attempted to harmonize its interpretation of the 21stCentury Cures Act with existing international criteria for software\u2014has also drawn concerns from some health care provider organizations. They argue that the guidance may exclude too many types of software from review and that FDA needs to clarify how the agency would apply it to specific products.58\nThis is particularly the case for CDS products\u2014including those that rely on AI\u2014developed and used by health care providers. Some health systems may be developing or piloting AI-driven CDS software for use within their own facility that might technically meet the definition of a medical device. The distinction between the practice of medicine\u2014which FDA does not regulate\u2014and a device is unclear in circumstances in which a software program is developed and implemented within a single health care system and is not sold to an outside party. The agency has not publicly stated its position on this issue; however, current regulations do exempt licensed practitioners who manufacture or alter devices solely for use in their practice from product registration requirements.59\nHospital accrediting bodies (such as the Joint Commission), standards-setting organizations (such as the Association for the Advancement of Medical Instrumentation), and government actors may need to fill this gap in oversight to ensure patient safety as these tools are more widely adopted.60For example, the Federal Trade Commission (FTC), which is responsible for protecting consumers and promoting fair market competition, published guidance in April 2020 for organizations using AI-enabled algorithms. Because algorithms that automate decision-making have the potential to produce negative or adverse outcomes for consumers, the guidance emphasizes the importance of using tools that are transparent, fair, robust, and explainable to the end consumer.61One year later, the FTC announced that it may take action against those organizations whose algorithms may be biased or inaccurate.62\nEmerging FDA proposals for SaMD regulation\nFDA officials have acknowledged that the rapid pace of innovation in the digital health field poses a significant challenge for the agency. They say new regulatory frameworks will be essential to allow the agency to ensure the safety and effectiveness of the devices on the market without unnecessarily slowing progress.63\nIn 2019, the agency began piloting an oversight framework called the Software Precertification Program, which, if fully implemented, would be a significant departure from its normal review process. Rather than reviewing devices individually, FDA would first evaluate the developer. If the organization meets certain qualifications and demonstrates it has rigorous processes to develop safe, effective devices, it would be able to undergo a significantly streamlined review process and make changes or even introduce products without going through premarket review. Nine companies participated in this pilot program. The lessons learned may help inform the development of a future regulatory model for software-based medical devices.64However, some members of Congress have questioned FDA\u2019s statutory authority to establish this program.65Legislation may be required before FDA can fully implement it, and its appeal to software developers is not yet clear.\nThe agency has also proposed a regulatory framework targeted to SaMD products that rely on an adaptive learning approach. Thus far, FDA has only cleared or approved AI devices that rely on a \u201clocked\u201d algorithm, which does not change over time unless it is updated by the developer. Adaptive algorithms, by contrast, have the potential to incorporate new data and \u201clearn\u201d in real time, meaning that the level of risk or performance of the product may also change rapidly. Given the speed and sometimes unpredictable nature of these changes, it can be difficult to determine when the SaMD\u2019s algorithm may require additional review by the agency to ensure that it is still safe and effective for its intended use.\nIn a 2019 white paper, FDA outlined a potential approach to addressing this question of adaptive learning. It is based on four general principles:66\nClear expectations on quality systems and good machine learning practices.As with any device manufacturer, FDA expects SaMD developers to have an established system to ensure that their device meets the relevant quality standards and conforms to regulations. In addition, a developer would need to implement established best practices for developing an algorithm, known as Good Machine Learning Practices (GMLP). This set of standards is still evolving, and may eventually need to be included as an amendment to current Good Manufacturing Practice requirements for devices.67FDA has recently stated it needs industry and stakeholder input to address outstanding questions on what these good practices look like for algorithm design, training, and testing.68\nPremarket assessment of SaMD products that require it.Under this framework, developers would have the option to submit a plan for future modifications, called a predetermined change control plan, as part of the initial premarket review of an SaMD that relies on AI/ML. This plan would include the types of anticipated modifications that may occur and the approach the developer would use to implement those changes and reduce the associated risks.\nRoutine monitoring of SaMD products by manufacturers to determine when an algorithm change requires FDA review.Under the current regulatory framework, many changes to an SaMD product would likely require the developer to file a new premarket submission. In the proposed approach, if modifications are made within the bounds of the predetermined change control plan, developers would need only to document those changes. If the changes are beyond the scope of the change control plan but do not lead to a new intended use of the device (for example, the developer makes the SaMD compatible with other sources of data, or incorporates a different type of data), then FDA may perform a review of the change control plan alone and approve a new version. However, if the modifications lead to a new intended use (for example, by expanding the target patient population from adults to children), then FDA would likely need to conduct an additional premarket review.\nTransparency and real-world performance monitoring.As part of this approach, FDA would expect a commitment from developers to adhere to certain principles of transparency and engage in ongoing performance monitoring. As such, developers would be expected to provide periodic reporting to the agency on implemented updates and performance metrics, among other requirements.\nThe proposed framework would be a significant shift in how FDA currently regulates devices, and\u2014as with the precertification program\u2014the agency has acknowledged that certain aspects of the framework may require congressional approval to implement.69Even if permission is granted, there are outstanding questions about how this framework would be implemented in practice and applied to specific devices. FDA is currently working on a series of follow-up documents that will provide further details on its proposed approach.70\nMost recently, the agency published the \u201cArtificial Intelligence/Machine Learning (AI/ML)-Based Software as a Medical Device (SaMD) Action Plan,\u201d outlining its intended next steps. These include updating its proposed framework and issuing draft guidance on the predetermined change control plan, encouraging harmonization among technology developers on the development of GMLP, and holding a public workshop on medical device labeling to support transparency to end users. In addition, the agency will support efforts to develop methods for the evaluation and improvement of ML algorithms, including how to identify and eliminate bias, and to work with stakeholders to advance real-world performance monitoring pilots.71\nRemaining questions and oversight gaps\nEspecially as the use of AI products in health care proliferates, FDA and other stakeholders will need to develop clear guidelines on the clinical evidence necessary to demonstrate the safety and effectiveness of such products and the extent to which product labels need to specify limitations on their performance and generalizability. As part of this effort, the agency could consider requiring developers to provide public information about the data used to validate and test AI devices so that end users can better understand their benefits and risks.\nFDA\u2019s recent SaMD Action Plan is a good step forward, but the agency will still need to clarify other key issues, including:\nWhen a modification to SaMD or an adaptive ML device requires premarket review. The draft guidance on the predetermined change control plan could be a critical part of this policy.\nWhether and how the Software Precertification Program can be extended beyond the pilot phase.\nThe distinction between software regulated by FDA and exempt software, which will turn heavily on the difference between informing clinical decisions and driving them.\nHow GMLP, when they are developed, will intersect with the current quality system regulations that apply to all devices.\nHow software updates and potential impacts on performance will be communicated to end users.\nIn addition, because there are products otherwise excluded from the definition of a medical device, another oversight body may need to play a role in ensuring patient safety, particularly for AI-enabled software not subject to FDA\u2019s authority. Further, for AI products used in the drug development process, FDA may need to provide additional guidance on the extent and type of evidence necessary to validate that products are working as intended.72\nTo fully seize the potential benefits that AI can add to the health care field while simultaneously ensuring the safety of patients, FDA may need to forge partnerships with a variety of stakeholders, including hospital accreditors, private technology firms, and other government actors such as the Office of the National Coordinator for Health Information Technology, which promulgates key standards for many software products, or the Centers for Medicare and Medicaid Services, which makes determinations about which technologies those insurance programs will cover. And, as previously mentioned, Congress may need to grant FDA additional authorities before the agency can implement some of its proposed policies, particularly as they relate to the precertification pilot.\nConclusion\nAI represents a transformational opportunity to improve patient outcomes, drive efficiency, and expedite research across health care. As such, health care providers, software developers, and researchers will continue to innovate and develop new AI products that test the current regulatory framework. FDA is attempting to meet these challenges and develop policies that can enable innovation while protecting public health, but there are many questions that the agency will need to address in order to ensure that this happens. As these policies evolve, legislative action may also be necessary to resolve the regulatory uncertainties within the sector.\nGlossary\nExplainability:The ability fordevelopers to explain in plain language how their data will be used.73\nGeneralizability:The accuracy with which results or findings can be transferred to other situations or people outside of those originally studied.74\nGood Machine Learning Practices(GMLP):AI/ML best practices (such as those for data management or evaluation), analogous to good software engineering practices or quality system practices.75\nMachine learning (ML):An AI technique that can be used to design and train software algorithms to learn from and act on data. These algorithms can be \u201clocked,\u201d so that their function does not change, or \u201cadaptive,\u201d meaning that their behavior can change over time.76\nSoftware as a Medical Device(SaMD):Defined by the International Medical Device Regulators Forum as \u201csoftware intended to be used for one or more medical purposes that perform these purposes without being part of a hardware medical device.\u201d77\nEndnotes\nExplore\nMore onFood & drug safety\nMore fromHealth Care Products\nMore inAdvance Health & Well-Being\nMORE FROM PEW\nIn Philadelphia, Declining Poverty Rate Among Latinos Offers an Incomplete Picture\nFounded in 1948, The Pew Charitable Trusts uses data to make a difference. Pew addresses the challenges of a changing world by illuminating issues, creating common ground, and advancing ambitious projects that lead to tangible progress.\nPew Research Centeris a nonpartisan fact tank that informs the public about the issues, attitudes and trends shaping the world, and a subsidiary of The Pew Charitable Trusts.", "cleaned_text_custom_parser_2": "\n\n \n

\n Topics:\n \n \n Advance Health & Well-Being\n \n \n \n \n Improve Federal Policy\n \n

\n Projects:\n \n Health Care Products\n \n

\n How FDA Regulates Artificial Intelligence in Medical Products\n

\n As technology evolves, oversight will need to keep pace\n

\n Issue Brief\n August 5, 2021\n

\n \n How FDA Regulates Artificial Intelligence Technologies in Medical Products (PDF)\n \n

\n SHARE\n

\n Downloads\n

\n \n How FDA Regulates Artificial Intelligence Technologies in Medical Products (PDF)\n \n

\n Picture Alliance\n Contributer via Getty Images\n

\n \n

\n Overview\n

\n Health care organizations are using artificial intelligence (AI)\u2014which the U.S. Food and Drug Administration defines as \u201cthe science and engineering of making intelligent machines\u201d\u2014for a growing range of clinical, administrative, and research purposes. This AI software can, for example, help health care providers diagnose diseases, monitor patients\u2019 health, or assist with rote functions such as scheduling patients.\n

\n Although AI offers unique opportunities to improve health care and patient outcomes, it also comes with potential challenges. AI-enabled products, for example, have sometimes resulted in inaccurate, even potentially harmful, recommendations for treatment.\n ^{\n 1\n}\n These errors can be caused by unanticipated sources of bias in the information used to build or train the AI, inappropriate weight given to certain data points analyzed by the tool, and other flaws.\n

\n The regulatory framework governing these tools is complex. FDA regulates some\u2014but not all\u2014AI-enabled products used in health care, and the agency plays an important role in ensuring the safety and effectiveness of those products under its jurisdiction. The agency is currently considering how to adapt its review process for AI-enabled medical devices that have the ability to evolve rapidly in response to new data, sometimes in ways that are difficult to foresee.\n ^{\n 2\n}\n

\n This brief describes current and potential uses of AI in health care settings and the challenges these technologies pose, outlines how and under what circumstances they are regulated by FDA, and highlights key questions that will need to be addressed to ensure that the benefits of these devices outweigh their risks. It will take a collective effort by FDA, Congress, technology developers, and the health care industry to ensure the safety and effectiveness of AI-enabled technology.\n

\n What is AI and how is it used in health care?\n

\n AI refers to the ability of a machine to perform a task that mimics human behavior, including problem-solving and learning.\n ^{\n 3\n}\n It can be used for a range of purposes, including automating tasks, identifying patterns in data, and synthesizing multiple sources of information. In health care, AI technologies are already used in fields that rely on image analysis, such as radiology and ophthalmology, and in products that process and analyze data from wearable sensors to detect diseases or infer the onset of other health conditions.\n ^{\n 4\n}\n

\n AI programs can also predict patient outcomes based on data collected from electronic health records, such as determining which patients may be at higher risk for disease or estimating who should receive increased monitoring. One such model identifies patients in the emergency room who may be at increased risk for developing sepsis based on factors such as vital signs and test results from electronic health records.\n ^{\n 5\n}\n Another hospital system has developed a model that aims to better predict which discharged patients are likely to be readmitted following their release compared with other risk-assessment tools.\n ^{\n 6\n}\n Other health care systems will likely follow suit in developing their own models as the technology becomes more accessible and well-established, and as federal regulations implement efforts to facilitate data exchange between electronic health record systems and mobile applications, a process known as interoperability.\n ^{\n 7\n}\n

\n Finally, AI can also play a role in research, including pharmaceutical development, combing through large sets of clinical data to improve a drug\u2019s design, predict its efficacy, and discover novel ways to treat diseases.\n ^{\n 8\n}\n The COVID-19 pandemic might help drive advances in AI in the clinical context, as hospitals and researchers have deployed it to support research, predict patient outcomes, and diagnose the disease.\n ^{\n 9\n}\n Some examples of AI products developed for use against COVID-19:\n ^{\n 10\n}\n

\n
\n COViage, a software prediction system, assesses whether hospitalized COVID-19 patients are at high risk of needing intubation.\n ^{\n 11\n}\n
\n
\n
\n CLEWICU System, prediction software that identifies which ICU COVID-19 patients are at risk for respiratory failure or low blood pressure.\n ^{\n 12\n}\n
\n
\n
\n Mount Sinai Health System developed an AI model that analyzes computed tomography (CT) scans of the chest and patient data to rapidly detect COVID-19.\n ^{\n 13\n}\n
\n
\n
\n Researchers at the University of Minnesota, along with Epic Systems and M Health Fairview, developed an AI tool that can evaluate chest X-rays to diagnose possible cases of COVID-19.\n ^{\n 14\n}\n
\n

\n How are AI products developed?\n

\n AI can be developed using a variety of techniques. In traditional, or rules-based, approaches, an AI program will follow human-prescribed instructions for how to process data and make decisions, such as being programmed to alert a physician each time a patient with high blood pressure should be prescribed medication.\n ^{\n 15\n}\n Rules-based approaches are usually grounded in established best practices, such as clinical practice guidelines or literature.\n ^{\n 16\n}\n On the other hand, machine learning (ML) algorithms\u2014also referred to as a data-based approach\u2014\u201clearn\u201d from numerous examples in a dataset without being explicitly programmed to reach a particular answer or conclusion.\n ^{\n 17\n}\n ML algorithms can learn to decipher patterns in patient data at scales larger than a human can analyze while also potentially uncovering previously unrecognized correlations.\n ^{\n 18\n}\n Algorithms may also work at a faster pace than a human. These capabilities could be especially useful in health care settings, which can provide continuous streams of data from sources, including patient medical records and clinical studies.\n ^{\n 19\n}\n

\n Most ML-driven applications use a supervised approach in which the data used to train and validate the algorithm is labeled in advance by humans; for example, a collection of chest X-rays taken of people who have lung cancer and those who do not, with the two groups identified for the AI software. The algorithm examines all examples within the training dataset to \u201clearn\u201d which features of a chest X-ray are most closely correlated with the diagnosis of lung cancer and uses that analysis to predict new cases. Developers then test the algorithm to see how generalizable it is; that is, how well it performs on a new dataset, in this case, a new set of chest X-rays. Further validation is required by the end user, such as the health care practice, to ensure that the algorithm is accurate in real-world settings. Unsupervised learning is also possible, in which an algorithm does not receive labeled data and instead infers underlying patterns within a dataset.\n ^{\n 20\n}\n

\n Challenges and risks with AI-enabled products\n

\n Like any digital health tool, AI models can be flawed, presenting risks to patient safety. These issues can stem from a variety of factors, including problems with the data used to develop the algorithm, the choices that developers make in building and training the model, and how the AI-enabled program is eventually deployed.\n

\n AI programs should be built and trained appropriately\n

\n AI algorithms need to be trained on large, diverse datasets to be generalizable across a variety of populations and to ensure that they are not biased in a way that affects their accuracy and reliability. These challenges can resemble those for other health care products. For example, if a drug is tested in a clinical trial population that is not sufficiently representative of the actual populations it will be used in, it will not work as well when implemented in real-world clinical settings. In AI, similarly, any model must be evaluated carefully to ensure that its performance can be applied across a diverse set of patients and settings.\n

\n However, such datasets are often difficult and expensive to assemble because of the fragmented U.S. health care system, characterized by multiple payers and unconnected health record systems. These factors can increase the propensity for error due to datasets that are incomplete or inappropriately merged from multiple sources.\n ^{\n 21\n}\n A 2020 analysis of data used to train image-based diagnostic AI systems found that approximately 70% of the studies that were included used data from three states, and that 34 states were not represented at all. Algorithms developed without considering geographic diversity, including variables such as disease prevalence and socioeconomic differences, may not perform as well as they should across a varied array of real-world settings.\n ^{\n 22\n}\n

\n The data collection challenges and the inequalities embedded within the health care system contribute to bias in AI programs that can affect product safety and effectiveness and reinforce the disparities that have led to improper or insufficient treatment for many populations, particularly minority groups.\n ^{\n 23\n}\n For example, cardiovascular disease risks in populations of races and ethnicities that are not White have been both overestimated and underestimated by algorithms trained with data from the Framingham Heart Study, which mostly involved White patients.\n ^{\n 24\n}\n Similarly, if an algorithm developed to help detect melanoma is trained heavily on images of patients with lighter skin tones, it may not perform as well when analyzing lesions on people of color, who already present with more advanced skin disease and face lower survival rates than White patients.\n ^{\n 25\n}\n

\n Bias can also occur when an algorithm developed in one setting, such as a large academic medical center, is applied in another, such as a small rural hospital with fewer resources. If not adapted and validated for its new context, an AI program may recommend treatments that are not available or appropriate in a facility with less access to specialists and cutting-edge technology.\n ^{\n 26\n}\n

\n Moreover, assembling sufficiently large patient datasets for AI-enabled programs can raise complex questions about data privacy and the ownership of personal health data. Protections to ensure that sensitive patient data remains anonymous are vital.\n ^{\n 27\n}\n Some health systems are sharing their patients\u2019 data with technology companies and digital startups to develop their AI-based programs, sometimes without those patients\u2019 knowledge. There is ongoing debate over whether patients should consent to having their data shared, and whether they should share in the profits from products that outside entities develop using their data.\n ^{\n 28\n}\n However, anonymizing patient data can pose its own challenges, as it can sometimes undermine efforts to ensure the representativeness of large datasets; if patient demographics are unknown to AI developers, then they may not be able to detect bias in the data.\n

\n Ensuring safe and effective use of AI-enabled products\n

\n Once an AI-enabled program has been developed, it must be used in a way that ensures that it consistently performs as expected. This can be a complex undertaking, depending on the purpose of the AI model and how it is updated.\n

\n ML algorithms, for example, fall along a spectrum from \u201clocked\u201d to \u201cadaptive\u201d (also referred to as \u201ccontinuous learning\u201d). In a locked algorithm, the same input will always produce the same result unless the developer updates the program. In contrast, an adaptive algorithm has the potential to update itself based on new data, meaning that the same input could generate different decisions and recommendations over time.\n ^{\n 29\n}\n Either type of algorithm presents its own challenges.\n

\n Locked algorithms can degrade as new treatments and clinical practices arise or as populations alter over time. These inevitable changes may make the real-world data entered into the AI program vastly different from its training data, leading the software to yield less accurate results. An adaptive algorithm could present an advantage in such situations, because it may learn to calibrate its recommendations in response to new data, potentially becoming more accurate than a locked model. However, allowing an adaptive algorithm to learn and adapt on its own also presents risks, including that it may infer patterns from biased practices or underperform in small subgroups of patients.\n ^{\n 30\n}\n

\n AI-enabled programs can also pose risks if they are not deployed appropriately and monitored carefully. One study found that a widely used algorithm disproportionately recommended White patients for high-risk care management programs, which provide intensive\u2014and often expensive\u2014services to people with complex health needs. Several health systems relied on the algorithm to identify patients who were most likely to benefit. However, the algorithm used higher health care costs as a proxy for medical need. Because Black patients are less likely to have access to care, even if they are insured, their health care costs tend to be lower. As a result, the algorithm systematically underestimated their health needs and excluded them from high-risk care programs.\n ^{\n 31\n}\n

\n Other challenges relate to the explainability of the output\u2014that is, how easy it is to explain to the end user how a program produced a certain result\u2014and the lack of transparency around how an AI-enabled program was developed. Some AI programs, for example, are referred to as \u201cblack-box\u201d models because the algorithms are derived from large datasets using complex techniques and reflect underlying patterns that may be too convoluted for a person, including the initial programmer, to understand. AI companies may also choose to keep their algorithms confidential, as proprietary information.\n ^{\n 32\n}\n Moreover, companies do not always publicly report detailed information on the datasets they use to develop or validate algorithms, limiting the ability of health care providers to evaluate how well the AI will perform for their patients. For example, a report examining companies\u2019 public summaries about their FDA-approved AI tools found that, of the 10 products approved for breast imaging, only one included a breakdown of the racial demographics in the dataset used to validate the algorithm. Breast cancer is significantly more likely to be fatal in Black women, who may be diagnosed at later stages of the disease and who experience greater barriers to care. Some or all of the AI devices in question may have been trained and validated on diverse patient populations, but the lack of public disclosure means that health care providers and patients might not have all the information they need to make informed decisions about the use of these products.\n ^{\n 33\n}\n

\n In addition, patients are often not aware when an AI program has influenced the course of their care; these tools could, for example, be part of the reason a patient does not receive a certain treatment or is recommended for a potentially unnecessary procedure.\n ^{\n 34\n}\n Although there are many aspects of health care that a patient may not fully understand, in a recent patient engagement meeting hosted by FDA, some committee members\u2014including patient advocates\u2014expressed a desire to be notified when an AI product is part of their care. This desire included knowing if the data the model was trained on was representative of their particular demographics, or if it had been modified in some way that changed its intended use.\n ^{\n 35\n}\n

\n Given the complexity of these products and the challenge of deploying them, health systems may need to recruit or train staff members with the technical skills to evaluate these models, understand their limitations, and implement them effectively. A provider\u2019s trust in\u2014and ability to correctly and appropriately use\u2014an AI tool is fundamental to its safety and effectiveness, and these human factors may vary significantly across institutions and even individuals.\n ^{\n 36\n}\n If providers do not understand how and why an algorithm arrived at a particular decision or result, they may struggle to interpret the result or apply it to a patient.\n

\n Software developers, health care providers, policymakers, and patients all have a role to play in addressing these various challenges. Regulatory agencies also may need to adapt their current oversight processes to keep pace with the rapid shifts underway in this field.\n

\n How and under what circumstances does FDA regulate AI products?\n

\n FDA is tasked with ensuring the safety and effectiveness of many AI-driven medical products. The agency largely regulates software based on its intended use and the level of risk to patients if it is inaccurate. If the software is intended to treat, diagnose, cure, mitigate, or prevent disease or other conditions, FDA considers it a medical device.\n ^{\n 37\n}\n Most products considered medical devices and that rely on AI/ML are categorized as Software as a Medical Device (SaMD).\n ^{\n 38\n}\n Examples of SaMD include software that helps detect and diagnose a stroke by analyzing MRI images, or computer-aided detection (CAD) software that processes images to aid in detecting breast cancer.\n ^{\n 39\n}\n Some consumer-facing products\u2014such as certain applications that run on a smartphone\u2014may also be classified as SaMD.\n ^{\n 40\n}\n By contrast, FDA refers to a computer program that is integral to the hardware of a medical device\u2014such as one that controls an X-ray panel\u2014as Software in a Medical Device.\n ^{\n 41\n}\n These products can also incorporate AI technologies.\n

\n Examples of FDA Cleared or Approved AI-Enabled Products\n

\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n

\n IDx-DR: Detects diabetic retinopathy\n

\n This software analyzes images of the eye to determine whether the patient should be\n referred to an eye professional because the images portray more than mild diabetic\n retinopathy or the patient should be rescreened in a year because the images were\n negative for more than mild diabetic retinopathy.\n ^{\n 42\n}\n

\n OsteoDetect: Detects and diagnoses wrist fractures\n

\n This software analyzes X-rays for signs of distal radius fracture and marks the location\n of the fracture to aid in detection and diagnosis.\n ^{\n 43\n}\n

\n ContaCT: Detects a possible stroke and notifies a specialist\n

\n This software analyzes CT images of the brain for indicators usually associated with\n a stroke, and immediately texts a specialist if a suspected large vessel blockage is\n identified, potentially involving the specialist sooner than the usual standard of care.\n ^{\n 44\n}\n

\n Guardian Connect System: Continuous glucose monitoring system\n

\n This product monitors glucose levels in the tissues of a diabetic patient, using a sensor\n inserted under the skin, either on an arm or on the abdomen. A transmitter processes and\n sends this information wirelessly to an application installed on a mobile device. Patients\n can use the program to monitor whether their glucose levels are too low or high.\n ^{\n 45\n}\n

\n Embrace2: Wearable seizure monitoring device\n

\n This product monitors physiological signals through a device worn on the wrist. If the\n technology senses activity that may indicate a seizure, it will send a command to a\n paired wireless device programmed to alert the patient\u2019s designated caregiver. The\n system will also record and store data from its sensors for future review by a health\n care professional.\n ^{\n 46\n}\n

\n FibriCheck: Mobile application to detect atrial fibrillation\n

\n This application uses either a smartphone camera or sensors in a smartwatch to\n analyze and record heart rhythms. This information, along with symptoms the patient\n is prompted to enter, are aggregated into a report along with next steps for the patient,\n if necessary.\n ^{\n 47\n}\n

\n As with any medical device, AI-enabled software is subject to FDA review based on its risk classification. Class I devices\u2014such as software that solely displays readings from a continuous glucose monitor\u2014 pose the lowest risk. Class II devices are considered to be moderate to high risk, and may include AI software tools that analyze medical images such as mammograms and flag suspicious findings for a radiologist to review.\n ^{\n 48\n}\n Most Class II devices undergo what is known as a 510(k) review (named for the relevant section of the Federal Food, Drug, and Cosmetic Act), in which a manufacturer demonstrates that its device is \u201csubstantially equivalent\u201d to an existing device on the market with the same intended use and technological characteristics.\n ^{\n 49\n}\n One study found that the majority of FDA-reviewed AI-based devices on the market have come through FDA\u2019s 510(k) pathway. However, the authors note that they relied on publicly available information, and because the agency does not require companies to categorize their devices as AI/ML-based in public documents, it is difficult to know the true number.\n ^{\n 50\n}\n

\n Alternatively, certain Class I and Class II device manufacturers may submit a De Novo request to FDA, which can be used for devices that are novel but whose safety and underlying technology are well understood, and which are therefore considered to be lower risk.\n ^{\n 51\n}\n Several AI-driven devices currently on the market\u2014such as IDx-DR, OsteoDetect, and ContaCT (see the text box, \u201cExamples of FDA Cleared or Approved AI-Enabled Products\u201d)\u2014are Class II devices that were reviewed through the De Novo pathway.\n ^{\n 52\n}\n

\n Class III devices pose the highest risk. They include products that are life-supporting, life-sustaining, or substantially important in preventing impairment of human health. These devices must undergo the full premarket approval process, and developers must submit clinical evidence that the benefits of the product outweigh the risks.\n ^{\n 53\n}\n The continuous glucose monitoring system, Guardian Connect system, was approved through a premarket approval.\n ^{\n 54\n}\n

\n Once a device is on the market, FDA takes a risk-based approach to determine whether it will require premarket review of any changes the developer makes. In general, each time a manufacturer significantly updates the software or makes other changes that would substantially affect the device\u2019s performance, the device may be subject to additional review by FDA, although the process for this evaluation differs depending on the device\u2019s risk classification and the nature of the change.\n

\n Exemptions From FDA Review\n

\n Congress excluded certain health-related software from the definition of a medical device in the 21\n ^{\n st\n}\n Century Cures Act of 2016.\n ^{\n 55\n}\n Exemptions in the law include software that:\n

\n
\n Is intended for administrative support of a health care facility.\n
\n
- \n
  \n Example: software for scheduling, practice and inventory management, or to process and maintain financial records.\n
  \n
\n
\n
\n Is intended for maintaining or encouraging a healthy lifestyle and is unrelated to the diagnosis, cure, mitigation, prevention, or treatment of a disease or condition.\n
\n
- \n
  \n Example: mobile applications that actively monitor exercise, provide daily motivational tips to reduce stress, or offer tools to promote or encourage healthy eating.\n
  \n
\n
\n
\n Is intended to serve as electronic patient records, including patient-provided information, and its function is not intended to interpret or analyze patient records (including imaging data) for the purpose of diagnosis, cure, mitigation, prevention, or treatment.\n
\n
- \n
  \n Examples: mobile applications that allow patients with a certain medical condition to record measurements or other events to share with their health care provider as part of a disease management plan, or that allow health care providers to access their patient\u2019s personal health record hosted on a web-based or other platform.\n
  \n
\n
\n
\n Is intended for transferring, storing, converting formats, or displaying data or results.\n
\n
- \n
  \n Example: software functions that display medical device data without modifying the data.\n
  \n
\n
\n
\n Is not intended to acquire, process, or analyze data from scanning or diagnostic devices such as MRIs or\n \n in vitro\n \n clinical tests, AND is used for the purpose of:\n
\n
1. \n
  \n Displaying, analyzing, or printing medical data, such as patient information or peer-reviewed clinical studies.\n
  \n
2. \n
  \n Supporting or providing medical recommendations to a health care professional, on the condition that the software allows the provider to independently review how those recommendations were made.\n
  \n
\n

\n
\n Example: certain clinical decision support software.\n
\n

\n Source: 21\n ^{\n st\n}\n Century Cures Act of 2016, Food and Drug Administration\n

\n Clinical decision support (CDS) software is a broad term that FDA defines as technologies that provide health care providers and patients with \u201cknowledge and person-specific information, intelligently filtered or presented at appropriate times to enhance health and health care.\u201d\n ^{\n 56\n}\n Studies have shown that CDS software can improve patient care.\n ^{\n 57\n}\n These products can have device and nondevice applications. To be exempt from the definition of device, and not regulated by the FDA, CDS software must meet criteria that Congress set in the 21\n ^{\n st\n}\n Century Cures Act of 2016. (See the text box, \u201cExemptions From FDA Review.\u201d)\n

\n Crucially, the CDS software must support or provide recommendations to health care providers as they make clinical decisions, but the software cannot be intended to\n replace a provider\u2019s independent judgment. That is, the software can inform decisions, but it cannot be intended as the driving factor behind them. Otherwise, the software must be regulated as a medical device by the agency. The distinction between\n \n informing\n \n and\n \n driving\n \n a decision can be difficult to assess and has proved challenging for FDA to describe as it attempts to implement the law. The agency released draft guidance in 2017 on how it would interpret those provisions with respect to CDS software. In response to feedback from product developers, which raised concerns that the agency was interpreting its authority too broadly, FDA officials revised and re-released the draft guidance in 2019. However, the 2019 guidance\u2014in which FDA attempted to harmonize its interpretation of the 21\n ^{\n st\n}\n Century Cures Act with existing international criteria for software\u2014has also drawn concerns from some health care provider organizations. They argue that the guidance may exclude too many types of software from review and that FDA needs to clarify how the agency would apply it to specific products.\n ^{\n 58\n}\n

\n This is particularly the case for CDS products\u2014including those that rely on AI\u2014developed and used by health care providers. Some health systems may be developing or piloting AI-driven CDS software for use within their own facility that might technically meet the definition of a medical device. The distinction between the practice of medicine\u2014which FDA does not regulate\u2014and a device is unclear in circumstances in which a software program is developed and implemented within a single health care system and is not sold to an outside party. The agency has not publicly stated its position on this issue; however, current regulations do exempt licensed practitioners who manufacture or alter devices solely for use in their practice from product registration requirements.\n ^{\n 59\n}\n

\n Hospital accrediting bodies (such as the Joint Commission), standards-setting organizations (such as the Association for the Advancement of Medical Instrumentation), and government actors may need to fill this gap in oversight to ensure patient safety as these tools are more widely adopted.\n ^{\n 60\n}\n For example, the Federal Trade Commission (FTC), which is responsible for protecting consumers and promoting fair market competition, published guidance in April 2020 for organizations using AI-enabled algorithms. Because algorithms that automate decision-making have the potential to produce negative or adverse outcomes for consumers, the guidance emphasizes the importance of using tools that are transparent, fair, robust, and explainable to the end consumer.\n ^{\n 61\n}\n One year later, the FTC announced that it may take action against those organizations whose algorithms may be biased or inaccurate.\n ^{\n 62\n}\n

\n Emerging FDA proposals for SaMD regulation\n

\n FDA officials have acknowledged that the rapid pace of innovation in the digital health field poses a significant challenge for the agency. They say new regulatory frameworks will be essential to allow the agency to ensure the safety and effectiveness of the devices on the market without unnecessarily slowing progress.\n ^{\n 63\n}\n

\n In 2019, the agency began piloting an oversight framework called the Software Precertification Program, which, if fully implemented, would be a significant departure from its normal review process. Rather than reviewing devices individually, FDA would first evaluate the developer. If the organization meets certain qualifications and demonstrates it has rigorous processes to develop safe, effective devices, it would be able to undergo a significantly streamlined review process and make changes or even introduce products without going through premarket review. Nine companies participated in this pilot program. The lessons learned may help inform the development of a future regulatory model for software-based medical devices.\n ^{\n 64\n}\n However, some members of Congress have questioned FDA\u2019s statutory authority to establish this program.\n ^{\n 65\n}\n Legislation may be required before FDA can fully implement it, and its appeal to software developers is not yet clear.\n

\n The agency has also proposed a regulatory framework targeted to SaMD products that rely on an adaptive learning approach. Thus far, FDA has only cleared or approved AI devices that rely on a \u201clocked\u201d algorithm, which does not change over time unless it is updated by the developer. Adaptive algorithms, by contrast, have the potential to incorporate new data and \u201clearn\u201d in real time, meaning that the level of risk or performance of the product may also change rapidly. Given the speed and sometimes unpredictable nature of these changes, it can be difficult to determine when the SaMD\u2019s algorithm may require additional review by the agency to ensure that it is still safe and effective for its intended use.\n

\n In a 2019 white paper, FDA outlined a potential approach to addressing this question of adaptive learning. It is based on four general principles:\n ^{\n 66\n}\n

\n
\n \n Clear expectations on quality systems and good machine learning practices.\n \n As with any device manufacturer, FDA expects SaMD developers to have an established system to ensure that their device meets the relevant quality standards and conforms to regulations. In addition, a developer would need to implement established best practices for developing an algorithm, known as Good Machine Learning Practices (GMLP). This set of standards is still evolving, and may eventually need to be included as an amendment to current Good Manufacturing Practice requirements for devices.\n ^{\n 67\n}\n FDA has recently stated it needs industry and stakeholder input to address outstanding questions on what these good practices look like for algorithm design, training, and testing.\n ^{\n 68\n}\n
\n
\n
\n \n Premarket assessment of SaMD products that require it.\n \n Under this framework, developers would have the option to submit a plan for future modifications, called a predetermined change control plan, as part of the initial premarket review of an SaMD that relies on AI/ML. This plan would include the types of anticipated modifications that may occur and the approach the developer would use to implement those changes and reduce the associated risks.\n
\n
\n
\n \n Routine monitoring of SaMD products by manufacturers to determine when an algorithm change requires FDA review.\n \n Under the current regulatory framework, many changes to an SaMD product would likely require the developer to file a new premarket submission. In the proposed approach, if modifications are made within the bounds of the predetermined change control plan, developers would need only to document those changes. If the changes are beyond the scope of the change control plan but do not lead to a new intended use of the device (for example, the developer makes the SaMD compatible with other sources of data, or incorporates a different type of data), then FDA may perform a review of the change control plan alone and approve a new version. However, if the modifications lead to a new intended use (for example, by expanding the target patient population from adults to children), then FDA would likely need to conduct an additional premarket review.\n
\n
\n
\n \n Transparency and real-world performance monitoring.\n \n As part of this approach, FDA would expect a commitment from developers to adhere to certain principles of transparency and engage in ongoing performance monitoring. As such, developers would be expected to provide periodic reporting to the agency on implemented updates and performance metrics, among other requirements.\n
\n

\n The proposed framework would be a significant shift in how FDA currently regulates devices, and\u2014as with the precertification program\u2014the agency has acknowledged that certain aspects of the framework may require congressional approval to implement.\n ^{\n 69\n}\n Even if permission is granted, there are outstanding questions about how this framework would be implemented in practice and applied to specific devices. FDA is currently working on a series of follow-up documents that will provide further details on its proposed approach.\n ^{\n 70\n}\n

\n Most recently, the agency published the \u201cArtificial Intelligence/Machine Learning (AI/ML)-Based Software as a Medical Device (SaMD) Action Plan,\u201d outlining its intended next steps. These include updating its proposed framework and issuing draft guidance on the predetermined change control plan, encouraging harmonization among technology developers on the development of GMLP, and holding a public workshop on medical device labeling to support transparency to end users. In addition, the agency will support efforts to develop methods for the evaluation and improvement of ML algorithms, including how to identify and eliminate bias, and to work with stakeholders to advance real-world performance monitoring pilots.\n ^{\n 71\n}\n

\n Remaining questions and oversight gaps\n

\n Especially as the use of AI products in health care proliferates, FDA and other stakeholders will need to develop clear guidelines on the clinical evidence necessary to demonstrate the safety and effectiveness of such products and the extent to which product labels need to specify limitations on their performance and generalizability. As part of this effort, the agency could consider requiring developers to provide public information about the data used to validate and test AI devices so that end users can better understand their benefits and risks.\n

\n FDA\u2019s recent SaMD Action Plan is a good step forward, but the agency will still need to clarify other key issues, including:\n

\n
\n When a modification to SaMD or an adaptive ML device requires premarket review. The draft guidance on the predetermined change control plan could be a critical part of this policy.\n
\n
\n
\n Whether and how the Software Precertification Program can be extended beyond the pilot phase.\n
\n
\n
\n The distinction between software regulated by FDA and exempt software, which will turn heavily on the difference between informing clinical decisions and driving them.\n
\n
\n
\n How GMLP, when they are developed, will intersect with the current quality system regulations that apply to all devices.\n
\n
\n
\n How software updates and potential impacts on performance will be communicated to end users.\n
\n

\n In addition, because there are products otherwise excluded from the definition of a medical device, another oversight body may need to play a role in ensuring patient safety, particularly for AI-enabled software not subject to FDA\u2019s authority. Further, for AI products used in the drug development process, FDA may need to provide additional guidance on the extent and type of evidence necessary to validate that products are working as intended.\n ^{\n 72\n}\n

\n To fully seize the potential benefits that AI can add to the health care field while simultaneously ensuring the safety of patients, FDA may need to forge partnerships with a variety of stakeholders, including hospital accreditors, private technology firms, and other government actors such as the Office of the National Coordinator for Health Information Technology, which promulgates key standards for many software products, or the Centers for Medicare and Medicaid Services, which makes determinations about which technologies those insurance programs will cover. And, as previously mentioned, Congress may need to grant FDA additional authorities before the agency can implement some of its proposed policies, particularly as they relate to the precertification pilot.\n

\n Conclusion\n

\n AI represents a transformational opportunity to improve patient outcomes, drive efficiency, and expedite research across health care. As such, health care providers, software developers, and researchers will continue to innovate and develop new AI products that test the current regulatory framework. FDA is attempting to meet these challenges and develop policies that can enable innovation while protecting public health, but there are many questions that the agency will need to address in order to ensure that this happens. As these policies evolve, legislative action may also be necessary to resolve the regulatory uncertainties within the sector.\n

\n Glossary\n

\n \n Explainability:\n \n The ability for\n developers to explain in plain language how their data will be used.\n ^{\n 73\n}\n

\n \n Generalizability:\n \n The accuracy with which results or findings can be transferred to other situations or people outside of those originally studied.\n ^{\n 74\n}\n

\n \n Good Machine Learning Practices\n \n \n (GMLP):\n \n AI/ML best practices (such as those for data management or evaluation), analogous to good software engineering practices or quality system practices.\n ^{\n 75\n}\n

\n \n Machine learning (ML):\n \n An AI technique that can be used to design and train software algorithms to learn from and act on data. These algorithms can be \u201clocked,\u201d so that their function does not change, or \u201cadaptive,\u201d meaning that their behavior can change over time.\n ^{\n 76\n}\n

\n \n Software as a Medical Device\n \n \n (SaMD):\n \n Defined by the International Medical Device Regulators Forum as \u201csoftware intended to be used for one or more medical purposes that perform these purposes without being part of a hardware medical device.\u201d\n ^{\n 77\n}\n

\n Endnotes\n

\n E.J. Topol, \u201cHigh-Performance Medicine: The Convergence of Human and Artificial Intelligence,\u201d\n \n Nature Medicine\n \n 25 (2019): 44\u201356,\n \n https://www.nature.com/articles/s41591-018-0300-7\n \n .\n
\n U.S. Food and Drug Administration, \u201cArtificial Intelligence/Machine Learning (AI/ML)-Based Software as a Medical Device (SaMD) Action Plan\u201d (2021),\n \n https://www.fda.gov/media/145022/download\n \n ; U.S. Food and Drug Administration, \u201cProposed Regulatory Framework for Modifications to Artificial Intelligence/Machine Learning (AI/ML)-Based Software as a Medical Device (SaMD)\u2014Discussion Paper and Request for Feedback\u201d (2019),\n \n https://www.fda.gov/media/122535/download\n \n .\n
\n G. Daniel et al., \u201cCurrent State and Near-Term Priorities for AI-Enabled Diagnostic Support Software in Health Care\u201d (Duke-Margolis Center for Health Policy, 2019),\n \n https://healthpolicy.duke.edu/sites/default/files/2019-11/dukemargolisaienableddxss.pdf\n \n .\n
\n K.-H. Yu, A.L. Beam, and I.S. Kohane, \u201cArtificial Intelligence in Healthcare,\u201d\n \n Nature Biomedical Engineering\n \n 2 (2018): 719\u201331,\n \n https://www.nature.com/articles/s41551-018-0305-z\n \n .\n
\n L. Malone, \u201cDuke\u2019s Augmented Intelligence System Helps Prevent Sepsis in the ED,\u201d Duke Health, March 10, 2020,\n \n https://physicians.dukehealth.org/articles/dukes-augmented-intelligence-system-helps-prevent-sepsis-ed\n \n .\n
\n M. Garrity, \u201cU of Maryland Medical Systems Develops Machine Learning Model to Better Predict Readmissions,\u201d June 7, 2019,\n \n https://www.beckershospitalreview.com/artificial-intelligence/u-of-maryland-medical-systems-develops-machine-learning-model-to-better-predict-readmissions.html\n \n .\n
\n HealthIT.gov, \u201cAbout ONC\u2019s Cures Act Final Rule,\u201d accessed April 5, 2021,\n \n https://www.healthit.gov/curesrule/overview/about-oncs-cures-act-final-rule\n \n .\n
\n J. Vamathevan et al., \u201cApplications of Machine Learning in Drug Discovery and Development,\u201d\n \n Nature Reviews Drug Discovery\n \n 18 (2019): 463\u201377,\n \n https://www.nature.com/articles/s41573-019-0024-5\n \n .\n
\n L. Richardson, \u201cArtificial Intelligence Has Helped to Guide Pandemic Response, but Requires Adequate Regulation,\u201d The Pew Charitable Trusts, March 11, 2021,\n \n https://www.pewtrusts.org/en/research-and-analysis/articles/2021/03/11/artificial-intelligence-has-helped-to-guide-pandemic-response-but-requires-adequate-regulation\n \n ; N. Arora, A.K. Banerjee, and M.L. Narasu, \u201cThe Role of Artificial Intelligence in Tackling COVID-19,\u201d\n \n Future Virology\n \n (2020),\n \n https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7692869/\n \n .\n
\n The first two products listed, COViage and the CLEWICU System, were granted Emergency Use Authorization (EUA) by FDA, which allows developers to market their products during a public health emergency without completing the agency\u2019s standard review process.\n
\n R. Robbins, \u201cFDA Issues Rare Emergency Authorization for an Algorithm Used to Inform COVID-19 Care,\u201d STAT+, Oct. 5, 2020,\n \n https://www.statnews.com/2020/10/05/dascena-algorithm-covid19-fda/\n \n ; RADM Denise M. Hinton, chief scientist, U.S. Food and Drug Administration, letter to Ms. Carol Gu, vice president, Operations, Dascena Inc., \u201cEmergency Use Authorization (EUA) for Emergency Use of the COViage Hemodynamic Instability and Respiratory Decompensation Prediction System (COViage),\u201d Sept. 24, 2020,\n \n https://www.fda.gov/media/142454/download\n \n ; Dascena Inc., \u201cDascena Receives FDA EUA for COVID-19 Hemodynamic Instability and Respiratory Decompensation Prediction System,\u201d Oct. 2, 2020,\n \n https://www.dascena.com/press-releases-old-dev/dascena-receives-fda-eua-for-covid-19-hemodynamic-instability-and-respiratory-decompensation-prediction-system\n \n .\n
\n N.P. Taylor, \u201cEmergency Authorization Granted to COVID-19 ICU Prediction Software,\u201d MedTech Dive, May 28, 2020,\n \n https://www.medtechdive.com/news/emergency-authorization-granted-to-covid-19-icu-prediction-software/578738/\n \n ; RADM Denise M. Hinton, chief scientist, U.S. Food and Drug Administration, letter to CLEW Medical Ltd., c/o Ms. Yarmela Pavlovic, Manatt, Phelps & Phillips LLP, \u201cEmergency Use Authorization (EUA) for Emergency Use of the CLEWICU System,\u201d May 26, 2020,\n \n https://www.fda.gov/media/138369/download\n \n ; CLEW, \u201cClew Receives FDA Emergency Use Authorization (EUA) for Its Predictive Analytics Platform in Support of COVID-19 Patients,\u201d June 16, 2020,\n \n https://clewmed.com/clew-receives-fda-emergency-use-authorization-eua-for-its-predictive-analytics-platform-in-support-of-covid-19-patients/\n \n .\n
\n Mount Sinai, \u201cMount Sinai First in U.S. To Use Artificial Intelligence to Analyze Coronavirus (COVID-19) Patients,\u201d May 19, 2020,\n \n https://www.mountsinai.org/about/newsroom/2020/mount-sinai-first-in-us-to-use-artificial-intelligence-to-analyze-coronavirus-covid19-patients-pr\n \n \n .\n \n
\n Epic, \u201cUniversity of Minnesota Develops AI Algorithm to Analyze Chest X-Rays for COVID-19,\u201d Oct. 1, 2020,\n \n https://www.epic.com/epic/post/university-minnesota-develops-ai-algorithm-analyze-chest-x-rays-covid-19\n \n ; University of Minnesota, \u201cUniversity of Minnesota Develops AI Algorithm to Analyze Chest X-Rays for COVID-19,\u201d Oct. 1, 2020,\n \n https://twin-cities.umn.edu/news-events/university-minnesota-develops-ai-algorithm-analyze-chest-x-rays-covid-19\n \n .\n
\n A. Rajkomar, J. Dean, and I. Kohane, \u201cMachine Learning in Medicine,\u201d\n \n The New England Journal of Medicine\n \n 380, no. 14 (2019),\n \n https://www.nejm.org/doi/full/10.1056/NEJMra1814259\n \n .\n
\n Daniel et al., \u201cCurrent State and Near-Term Priorities for AI-Enabled Diagnostic Support Software in Health Care.\u201d\n
\n Ibid.; Rajkomar, Dean, and Kohane, \u201cMachine Learning in Medicine.\u201d\n
\n Rajkomar, Dean, and Kohane, \u201cMachine Learning in Medicine\u201d; Yu, Beam, and Kohane, \u201cArtificial Intelligence in Healthcare.\u201d\n
\n U.S. Food and Drug Administration, \u201cArtificial Intelligence and Machine Learning in Software as a Medical Device,\u201d last modified Jan. 12, 2021,\n \n https://www.fda.gov/medical-devices/software-medical-device-samd/artificial-intelligence-and-machine-learning-software-medical-device\n \n .\n
\n U.S. Food and Drug Administration, \u201cExecutive Summary for the Patient Engagement Advisory Committee Meeting\u201d (2020),\n \n https://www.fda.gov/media/142998/download\n \n ; M. Matheny et al., \u201cArtificial Intelligence in Health Care: The Hope, the Hype, the Promise, the Peril\u201d (National Academy of Medicine, 2020),\n \n https://nam.edu/wp-content/uploads/2019/12/AI-in-Health-Care-PREPUB-FINAL.pdf\n \n ; Yu, Beam, and Kohane, \u201cArtificial Intelligence in Healthcare.\u201d\n
\n W. Nicholson Price II, \u201cRisks and Remedies for Artificial Intelligence in Health Care,\u201d The Brookings Institution, Nov. 14, 2019,\n \n https://www.brookings.edu/research/risks-and-remedies-for-artificial-intelligence-in-health-care/\n \n .\n
\n A. Kaushal, R. Altman, and C. Langlotz, \u201cGeographic Distribution of U.S. Cohorts Used to Train Deep Learning Algorithms,\u201d\n \n Journal of the American Medical Association\n \n 324, no. 12 (2020),\n \n https://jamanetwork.com/journals/jama/article-abstract/2770833\n \n .\n
\n Nicholson Price II, \u201cRisks and Remedies for Artificial Intelligence in Health Care.\u201d\n
\n D.S. Char, N.H. Shah, and D. Magnus, \u201cImplementing Machine Learning in Health Care\u2014Addressing Ethical Challenges,\u201d\n \n New England Journal of Medicine\n \n 378, no. 11 (2018): 981\u201383,\n \n https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5962261/\n \n .\n
\n A.S. Adamson and A. Smith, \u201cMachine Learning and Health Care Disparities in Dermatology,\u201d\n \n JAMA Dermatology\n \n 154, no. 11 (2018): 1247-48,\n \n https://jamanetwork.com/journals/jamadermatology/article-abstract/2688587\n \n .\n
\n W.N.P. II, \u201cMedical AI and Contextual Bias,\u201d\n \n University of Michigan Public Law Research Paper No. 632; Harvard Journal of Law & Technology\n \n 33, no. 66 (2019),\n \n https://papers.ssrn.com/sol3/papers.cfm?abstract_id=3347890#\n \n .\n
\n C. Ross, \u201cAt Mayo Clinic, Sharing Patient Data With Companies Fuels AI Innovation\u2014and Concerns About Consent,\u201d STAT+, June 3, 2020,\n \n https://www.statnews.com/2020/06/03/mayo-clinic-patient-data-fuels-artificial-intelligence-consent-concerns/\n \n ; S. Gerke, T. Minssen, and G. Cohen, \u201cEthical and Legal Challenges of Artificial Intelligence-Driven Healthcare,\u201d\n \n Artificial Intelligence in Healthcare\n \n (2020): 295\u2013336,\n \n https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7332220/\n \n .\n
\n Ross, \u201cAt Mayo Clinic, Sharing Patient Data with Companies Fuels AI Innovation\u2014and Concerns About Consent.\u201d\n
\n U.S. Food and Drug Administration, \u201cProposed Regulatory Framework for Modifications to Artificial Intelligence/Machine Learning (AI/ML)-Based Software as a Medical Device (SaMD)\u201d (2021),\n \n https://www.fda.gov/files/medical devices/published/US-FDA-Artificial-Intelligence-and-Machine-Learning-Discussion-Paper.pdf\n \n ; Daniel et al., \u201cCurrent State and Near-Term Priorities for AI-Enabled Diagnostic Support Software in Health Care.\u201d\n
\n Daniel et al., \u201cCurrent State and Near-Term Priorities for AI-Enabled Diagnostic Support Software in Health Care.\u201d\n
\n Z. Obermeyer et al., \u201cDissecting Racial Bias in an Algorithm Used to Manage the Health of Populations,\u201d\n \n Science\n \n 366, no. 6464 (2019): 447-53,\n \n https://science.sciencemag.org/content/366/6464/447\n \n ; C.K. Johnson, \u201cRacial Bias in Health Care Software Aids Whites Over Blacks,\u201d The Seattle Times, Oct. 25, 2019,\n \n https://www.seattletimes.com/seattle-news/health/racial-bias-in-health-care-software-aids-whites-over-blacks/\n \n .\n
\n S. Samuel, \u201cAI Can Now Outperform Doctors at Detecting Breast Cancer. Here\u2019s Why It Won\u2019t Replace Them,\u201d Vox, Jan. 3, 2020,\n \n https://www.vox.com/future-perfect/2020/1/3/21046574/ai-google-breast-cancer-mammogram-deepmind\n \n ; W.N.P. II, \u201cRegulating Black-Box Medicine,\u201d\n \n 116 Michigan Law Review 42\n \n (2017),\n \n https://papers.ssrn.com/sol3/papers.cfm?abstract_id=2938391\n \n .\n
\n C. Ross, \u201cCould AI Tools for Breast Cancer Worsen Disparities? Patchy Public Data in FDA Filings Fuel Concern,\u201d STAT+, Feb. 11, 2021,\n \n https://www.statnews.com/2021/02/11/breast-cancer-disparities-artificial-intelligence-fda/\n \n ; Centers for Disease Control and Prevention, \u201cPatterns and Trends in Age-Specific Black-White Differences in Breast Cancer Incidence and Mortality\u2014United States, 1999\u20132014,\u201d\n \n Morbidity and Mortality Weekly Report\n \n 65, no. 40 (2016): 1093\u201398,\n \n https://www.cdc.gov/mmwr/volumes/65/wr/mm6540a1.htm?CDC_AA_refVal=https%3A%2F%2Fwww.cdc.gov%2Fcancer%2Fdcpc%2Fresearch%2Farticles%2Fbreast_cancer_rates_women.htm\n \n .\n
\n R. Robbins and E. Brodwin, \u201cAn Invisible Hand: Patients Aren\u2019t Being Told About the AI Systems Advising Their Care,\u201d July 15, 2020,\n \n https://www.statnews.com/2020/07/15/artificial-intelligence-patient-consent-hospitals/\n \n .\n
\n C. Ross, \u201cBias, Consent, and Data Transparency: What Patients Want the FDA to Consider About AI in Medicine,\u201d STAT+, Oct. 26, 2020,\n \n https://www.statnews.com/2020/10/26/artificial-intelligence-bias-fda-patients/\n \n .\n
\n E. Brodwin, \u201c\u2018It\u2019s Really on Them to Learn\u2019: How the Rapid Rollout of AI Tools Has Fueled Frustration Among Clinicians,\u201d Dec. 17, 2020,\n \n https://www.statnews.com/2020/12/17/artificial-intelligence-doctors-nurses-frustrated-ai-hospitals/\n \n .\n
\n U.S. Food and Drug Administration, \u201cProposed Regulatory Framework for Modifications to Artificial Intelligence/Machine Learning (AI/ML)-Based Software as a Medical Device (SaMD).\u201d\n
\n U.S. Food and Drug Administration, \u201cExecutive Summary for the Patient Engagement Advisory Committee Meeting.\u201d\n
\n U.S. Food and Drug Administration, \u201cWhat Are Examples of Software as a Medical Device?\u201d last modified Dec. 6, 2017,\n \n https://www.fda.gov/medical-devices/software-medical-device-samd/what-are-examples-software-medical-device\n \n ; Deloitte, \u201cSoftware as a Medical Device,\u201d accessed April 5, 2021,\n \n https://www2.deloitte.com/us/en/pages/public-sector/articles/software-as-a-medical-device-fda.html\n \n .\n
\n T. Minssen et al., \u201cRegulatory Responses to Medical Machine Learning,\u201d\n \n Journal of Law and the Biosciences\n \n (2020),\n \n https://academic.oup.com/jlb/advance-article/doi/10.1093/jlb/lsaa002/5817484\n \n .\n
\n U.S. Food and Drug Administration, \u201cSoftware as a Medical Device (SaMD),\u201d last modified Dec. 4, 2018,\n \n https://www.fda.gov/medical-devices/digital-health-center-excellence/software-medical-device-samd\n \n ; Deloitte, \u201cSoftware as a Medical Device.\u201d\n
\n U.S. Food and Drug Administration, \u201cFDA Permits Marketing of Artificial Intelligence-Based Device to Detect Certain Diabetes-Related Eye Problems,\u201d April 11, 2018,\n \n https://www.fda.gov/news-events/press-announcements/fda-permits-marketing-artificial-intelligence-based-device-detect-certain-diabetes-related-eye\n \n ; Digital Diagnostics, \u201cIDx-DR,\u201d accessed April 5, 2021,\n \n https://dxs.ai/products/idx-dr/idx-dr-overview/\n \n .\n
\n U.S. Food and Drug Administration, \u201cFDA Permits Marketing of Artificial Intelligence Algorithm for Aiding Providers in Detecting Wrist Fractures,\u201d May 24, 2018,\n \n https://www.fda.gov/news-events/press-announcements/fda-permits-marketing-artificial-intelligence-algorithm-aiding-providers-detecting-wrist-fractures\n \n ; U.S. Food and Drug Administration, \u201cEvaluation of Automatic Class III Designation for Osteodetect\u201d (\n \n https://www.accessdata.fda.gov/cdrh_docs/reviews/DEN180005.pdf\n \n .\n
\n U.S. Food and Drug Administration, \u201cFDA Permits Marketing of Clinical Decision Support Software for Alerting Providers of a Potential Stroke in Patients,\u201d Feb. 13, 2018,\n \n https://www.fda.gov/news-events/press-announcements/fda-permits-marketing-clinical-decision-support-software-alerting-providers-potential-stroke\n \n ; U.S. Food and Drug Administration, \u201cEvaluation of Automatic Class III Designation for Contact,\u201d\n \n https://www.accessdata.fda.gov/cdrh_docs/reviews/DEN170073.pdf\n \n .\n
\n U.S. Food and Drug Administration, \u201cGuardian Connect System\u2014P160007,\u201d last modified April 8, 2018,\n \n https://www.fda.gov/medical-devices/recently-approved-devices/guardian-connect-system-p160007\n \n ; The Medical Futurist, \u201cMedtronic\u2019s Smart Continuous Glucose Monitoring System Rolls Out This Summer,\u201d March 21, 2018,\n \n https://medicalfuturist.com/medtronics-smart-continuous-glucose-monitoring-system-rolls-summer/\n \n ; Medtronic, \u201cThe Guardian\u2122 Connect System,\u201d accessed April 5, 2021,\n \n https://www.medtronicdiabetes.com/products/guardian-connect-continuous-glucose-monitoring-system\n \n .\n
\n Empatica, \u201cIndications for Use and Safety Information,\u201d accessed April 5, 2021,\n \n https://www.empatica.com/embrace-IFU\n \n ; Empatica, \u201cHow Does Embrace2 Work? Watch Our Animated Video!\u201d June 7, 2019,\n \n https://www.empatica.com/blog/how-does-embrace2-work-watch-our-animated-video.html\n \n
\n I.V. Loon, \u201cFibriCheck Receives FDA Clearance for Its Digital Heart Rhythm Monitor,\u201d FibriCheck, Oct. 8, 2018,\n \n https://www.fibricheck.com/fibricheck-receives-fda-clearance-for-its-digital-heart-rhythm-monitor/\n \n .\n
\n Daniel et al., \u201cCurrent State and Near-Term Priorities for AI-Enabled Diagnostic Support Software in Health Care\u201d; for example: T. Mills, director, Division of Radiological Health, Office of In Vitro Diagnostics and Radiological Health, Center for Devices and Radiological Health, letter to Kevin Harris, CEO, CureMetrix Inc., \u201cReply to Section 510(k) Premarket Notification of Intent,\u201d March 8, 2019,\n \n http://www.accessdata.fda.gov/cdrh_docs/pdf18/K183285.pdf\n \n .\n
\n U.S. Food and Drug Administration, \u201cOverview of Device Regulation,\u201d last modified Sept. 4, 2020,\n \n https://www.fda.gov/medical-devices/device-advice-comprehensive-regulatory-assistance/overview-device-regulation\n \n ; U.S. Food and Drug Administration, \u201cStep 3: Pathway to Approval,\u201d last modified Feb. 9, 2018,\n \n https://www.fda.gov/patients/device-development-process/step-3-pathway-approval\n \n .\n
\n S. Benjamens, P. Dhunnoo, and B. Mesk\u00f3, \u201cThe State of Artificial Intelligence-Based FDA-Approved Medical Devices and Algorithms: An Online Database,\u201d\n \n npj Digital Medicine,\n \n Vol.3, no. 118 (2018),\n \n https://www.nature.com/articles/s41746-020-00324-0\n \n .\n
\n U.S. Food and Drug Administration, \u201cStep 3: Pathway to Approval\u201d; Daniel et al., \u201cCurrent State and Near-Term Priorities for AI-Enabled Diagnostic Support Software in Health Care.\u201d\n
\n U.S. Food and Drug Administration, \u201cDe Novo Classification Request for IDx-DR\u201d (2018),\n \n https://www.accessdata.fda.gov/cdrh_docs/reviews/DEN180001.pdf\n \n ; U.S. Food and Drug Administration, \u201cEvaluation of Automatic Class III Designation for Osteodetect\u201d; U.S. Food and Drug Administration, \u201cEvaluation of Automatic Class III Designation for Contact.\u201d\n
\n J. Jin, \u201cFDA Authorization of Medical Devices,\u201d\n \n JAMA\n \n 311, no. 4 (2014): 435,\n \n https://jamanetwork.com/journals/jama/fullarticle/1817798\n \n ; U.S. Food and Drug Administration, \u201cStep 3: Pathway to Approval.\u201d\n
\n C.H. Lias, director, Division of Chemistry and Toxicology Devices, Office of In Vitro Diagnostics and Radiological Health, Center for Devices and Radiological Health, letter to Liane Miller, Medtronic MiniMed, Regulatory Affairs Manager, \u201cPremarket Approval Application (PMA) Review,\u201d\n \n https://www.accessdata.fda.gov/cdrh_docs/pdf16/P160007a.pdf\n \n .\n
\n Section 520(o) of the Federal Food, Drug, and Cosmetic Act; United States Public Law 114-255\u201421st Century Cures Act (2016),\n \n https://www.congress.gov/114/plaws/publ255/PLAW-114publ255.pdf\n \n ; U.S. Food and Drug Administration, \u201cChanges to Existing Medical Software Policies Resulting from Section 3060 of the 21st Century Cures Act\u2014Guidance for Industry and Food and Drug Administration Staff\u201d (Sept. 27, 2019),\n \n https://www.fda.gov/media/109622/download\n \n .\n
\n U.S. Food and Drug Administration, \u201cClinical Decision Support Software\u2014Draft Guidance for Industry and Food and Drug Administration Staff\u201d (2019),\n \n https://www.fda.gov/media/109618/download\n \n .\n
\n For example: J.M. Sperl-Hillen et al., \u201cClinical Decision Support Directed to Primary Care Patients and Providers Reduces Cardiovascular Risk: A Randomized Trial,\u201d\n \n Journal of the American Medical Informatics Association\n \n 25, no. 9 (2018): 1137-46,\n \n https://pubmed.ncbi.nlm.nih.gov/29982627/\n \n .\n
\n D. Lim, \u201cIndustry Blasts FDA Clinical Decision Software Draft,\u201d Healthcare Dive, Feb. 7, 2018,\n \n https://www.healthcaredive.com/news/industry-blasts-fda-clinical-decision-software-draft/516523/\n \n ; D. Lim, \u201cLatest FDA Clinical Decision Support Software Draft a Step Forward, Industry Says,\u201d Medtech Dive, Jan. 7, 2020,\n \n https://www.medtechdive.com/news/latest-fda-clinical-decision-support-software-draft-a-step-forward-industr/569927/\n \n .\n
\n U.S. Food and Drug Administration, 21 CFR 807.65(d) (2020),\n \n https://www.accessdata.fda.gov/scripts/cdrh/cfdocs/cfcfr/cfrsearch.cfm?fr=807.65\n \n .\n
\n Nicholson Price II, \u201cRisks and Remedies for Artificial Intelligence in Health Care.\u201d\n
\n A. Smith, \u201cUsing Artificial Intelligence and Algorithms,\u201d Federal Trade Commission, accessed April 8, 2020,\n \n https://www.ftc.gov/news-events/blogs/business-blog/2020/04/using-artificial-intelligence-algorithms\n \n .\n
\n E. Jillson, \u201cAiming for Truth, Fairness, and Equity in Your Company\u2019s Use of AI,\u201d Federal Trade Commission, April 19, 2021,\n \n https://www.ftc.gov/news-events/blogs/business-blog/2021/04/aiming-truth-fairness-equity-your-companys-use-ai\n \n .\n
\n U.S. Food and Drug Administration, \u201cStatement from FDA Commissioner Scott Gottlieb, M.D., and Center for Devices and Radiological Health Director Jeff Shuren, M.D., J.D., on Agency Efforts to Work with Tech Industry to Spur Innovation in Digital Health,\u201d Sept. 12, 2018,\n \n https://www.fda.gov/news-events/press-announcements/statement-fda-commissioner-scott-gottlieb-md-and-center-devices-and-radiological-health-director\n \n ; U.S. Food and Drug Administration, \u201cStatement from FDA Commissioner Scott Gottlieb, M.D., On Steps Toward a New, Tailored Review Framework for Artificial Intelligence-Based Medical Devices,\u201d April 02, 2019,\n \n https://www.fda.gov/news-events/press-announcements/statement-fda-commissioner-scott-gottlieb-md-steps-toward-new-tailored-review-framework-artificial\n \n ; U.S. Food and Drug Administration, \u201cFDA Launches the Digital Health Center of Excellence,\u201d Sept. 22, 2020,\n \n https://www.fda.gov/news-events/press-announcements/fda-launches-digital-health-center-excellence\n \n .\n
\n U.S. Food and Drug Administration, \u201cDigital Health Software Precertification (Pre-Cert) Program,\u201d Sept. 14, 2020,\n \n https://www.fda.gov/medical-devices/digital-health-center-excellence/digital-health-software-precertification-pre-cert-program\n \n .\n
\n Senator Elizabeth Warren, Senator Patty Murray, and Senator Tina Smith, letter to Scott Gottlieb, commissioner, U.S. Food and Drug Administration, and Jeffrey Shuren, director, Center for Devices and Radiological Health, U.S. Food and Drug Administration, \u201cLetter to FDA on Regulation of Software as Medical Device,\u201d Oct. 10, 2018,\n \n https://www.warren.senate.gov/imo/media/doc/2018.10.10%20Letter%20to%20FDA%20on%20regulation%20of%20sofware%20as%20medical%20device.pdf\n \n .\n
\n U.S. Food and Drug Administration, \u201cArtificial Intelligence and Machine Learning in Software as a Medical Device\u201d; U.S. Food and Drug Administration, \u201cProposed Regulatory Framework for Modifications to Artificial Intelligence/Machine Learning (AI/ML)-Based Software as a Medical Device (SaMD).\u201d\n
\n U.S. Food and Drug Administration, 21 CFR 820 (2020),\n \n https://www.accessdata.fda.gov/scripts/cdrh/cfdocs/cfcfr/CFRSearch.cfm?CFRPart=820&showFR=1&subpartNode=21:8.0.1.1.12.1\n \n .\n
\n G. Slabodkin, \u201cFDA AI-Machine Learning Strategy Remains Work in Progress,\u201d Medtech Dive, accessed Sept. 14, 2020,\n \n https://www.medtechdive.com/news/fda-ai-machine-learning-strategy-remains-work-in-progress/585146/\n \n .\n
\n U.S. Food and Drug Administration, \u201cProposed Regulatory Framework for Modifications to Artificial Intelligence/Machine Learning (AI/ML)-Based Software as a Medical Device (SaMD)\u2014Discussion Paper and Request for Feedback\u201d; U.S. Food and Drug Administration, \u201cDeveloping the Software Precertification Program: Summary of Learnings and Ongoing Activities\u201d (2020), https://www.fda.gov/media/142107/download.\n
\n Slabodkin, \u201cFDA AI-Machine Learning Strategy Remains Work in Progress.\u201d\n
\n U.S. Food and Drug Administration, \u201cFDA Releases Artificial Intelligence/Machine Learning Action Plan,\u201d Jan. 12, 2021,\n \n https://www.fda.gov/news-events/press-announcements/fda-releases-artificial-intelligencemachine-learning-action-plan\n \n ; U.S. Food and Drug Administration, \u201cArtificial Intelligence/Machine Learning (AI/ML)-Based Software as a Medical Device (SaMD) Action Plan.\u201d\n
\n U.S. Government Accountability Office and National Academy of Medicine, \u201cArtificial Intelligence in Health Care Benefits and Challenges of Machine Learning in Drug Development\u201d (2019),\n \n www.gao.gov/assets/gao-20-215sp.pdf\n \n .\n
\n U.S. Food and Drug Administration, \u201cExecutive Summary for the Patient Engagement Advisory Committee Meeting.\u201d\n
\n Ibid.\n
\n U.S. Food and Drug Administration, \u201cProposed Regulatory Framework for Modifications to Artificial Intelligence/Machine Learning (AI/ML)-Based Software as a Medical Device (SaMD) \u2014Discussion Paper and Request for Feedback.\u201d\n
\n U.S. Food and Drug Administration, \u201cArtificial Intelligence and Machine Learning in Software as a Medical Device.\u201d\n
\n International Medical Device Regulators Forum, \u201cSoftware as a Medical Device (SaMD): Key Definitions\u201d (2013),\n \n http://www.imdrf.org/docs/imdrf/final/technical/imdrf-tech-131209-samd-key-definitions-140901.pdf\n \n ; U.S. Food and Drug Administration, \u201cSoftware as a Medical Device (SaMD).\u201d\n