{ "title": "Edictai article sources: What ten randomized trials say about AI tutors", "license": "CC BY 4.0", "article": "ai-tutoring-randomized-trials", "records": [ { "note": "1", "kind": "study", "jurisdiction": "", "identifier": "", "title": "AI tutoring outperforms in-class active learning: an RCT introducing a novel research-based design in an authentic educational setting", "status_or_publisher": "Scientific Reports", "locator": "Results, Learning gains", "quote": "The median learning gains for students, relative to the pre-test baseline (M = 2.75, N = 316), in the AI-tutored group were over double", "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12179260/" }, { "note": "2", "kind": "study", "jurisdiction": "", "identifier": "", "title": "AI tutoring outperforms in-class active learning: an RCT introducing a novel research-based design in an authentic educational setting", "status_or_publisher": "Scientific Reports", "locator": "Results, Time on task", "quote": "70% of students in the AI group spent less than 60 minutes on task, while 30% spent more than 60 minutes on task. The median time on task for students in the AI group was 49 minutes.", "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12179260/" }, { "note": "3", "kind": "study", "jurisdiction": "", "identifier": "", "title": "Effective and Scalable Math Support: Experimental Evidence on the Impact of an AI-Math Tutor in Ghana", "status_or_publisher": "arXiv", "locator": "Abstract", "quote": "We find that the math growth scores were substantially higher for the treatment group and statistically significant (p < 0.001). The effect size of 0.36 is considered large in the context of educational interventions", "source_url": "https://arxiv.org/pdf/2402.09809" }, { "note": "4", "kind": "study", "jurisdiction": "", "identifier": "", "title": "From Chalkboards to Chatbots: Evaluating the Impact of Generative AI on Learning Outcomes in Nigeria", "status_or_publisher": "World Bank", "locator": "Abstract", "quote": "The intervention demonstrated a significant improvement of 0.31 standard deviation on an assessment that included English topics aligned with the Nigerian curriculum, knowledge of artificial intelligence and digital skills.", "source_url": "https://documents.worldbank.org/curated/en/099548105192529324/pdf/IDU-c09f40d8-9ff8-42dc-b315-591157499be7.pdf" }, { "note": "5", "kind": "study", "jurisdiction": "", "identifier": "", "title": "From Chalkboards to Chatbots: Evaluating the Impact of Generative AI on Learning Outcomes in Nigeria", "status_or_publisher": "World Bank", "locator": "Introduction, p. 2", "quote": "We calculate an ITT effect of being selected for the program on the performance in the third-term exam of 0.21 standard deviations.", "source_url": "https://documents.worldbank.org/curated/en/099548105192529324/pdf/IDU-c09f40d8-9ff8-42dc-b315-591157499be7.pdf" }, { "note": "6", "kind": "study", "jurisdiction": "", "identifier": "", "title": "Tutor CoPilot: A Human-AI Approach for Scaling Real-Time Expertise", "status_or_publisher": "arXiv", "locator": "Abstract", "quote": "students whose tutors used Tutor CoPilot were 4 percentage points more likely to master mathematics topics (p<.01)", "source_url": "https://arxiv.org/pdf/2410.03017" }, { "note": "7", "kind": "study", "jurisdiction": "", "identifier": "", "title": "Tutor CoPilot: A Human-AI Approach for Scaling Real-Time Expertise", "status_or_publisher": "arXiv", "locator": "Results, RQ2", "quote": "lower-rated tutors experienced a 9 p.p. increase in students passing their exit ticket (56%→ 65%student passing rate from control to treatment). In Figure 2b, less experienced tutors showed a 7 p.p. increase (61%→ 68%).", "source_url": "https://arxiv.org/pdf/2410.03017" }, { "note": "8", "kind": "study", "jurisdiction": "", "identifier": "", "title": "Generative AI without guardrails can harm learning: Evidence from high school mathematics", "status_or_publisher": "Proceedings of the National Academy of Sciences", "locator": "Abstract", "quote": "having GPT-4 access while solving problems significantly improves performance (48% improvement in grades for GPT Base and 127% for GPT Tutor)", "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12232635/" }, { "note": "9", "kind": "study", "jurisdiction": "", "identifier": "", "title": "Generative AI without guardrails can harm learning: Evidence from high school mathematics", "status_or_publisher": "Proceedings of the National Academy of Sciences", "locator": "Main results", "quote": "on the exam, students in the GPT Base arm perform statistically significantly worse than students in the control arm by 17%; this negative effect is essentially eradicated in the GPT Tutor arm, though we still do not observe a positive effect.", "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12232635/" }, { "note": "10", "kind": "study", "jurisdiction": "", "identifier": "", "title": "Generative AI without guardrails can harm learning: Evidence from high school mathematics", "status_or_publisher": "Proceedings of the National Academy of Sciences", "locator": "Introduction", "quote": "we find evidence that students do not perceive any reduction in their learning or subsequent performance as a consequence of copying solutions", "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12232635/" }, { "note": "11", "kind": "study", "jurisdiction": "", "identifier": "", "title": "One Click Away: AI Tutoring with Khanmigo in a Two-Year School Experiment", "status_or_publisher": "Annenberg Institute at Brown University (EdWorkingPapers)", "locator": "Abstract", "quote": "Assignment raises math achievement by 1.3 national percentile ranks per term, or about 0.06 to 0.08 standard deviations over a school year; the implied effect of a full year of active participation reaches 0.14 standard deviations.", "source_url": "https://edworkingpapers.com/ai26-1551" }, { "note": "12", "kind": "study", "jurisdiction": "", "identifier": "", "title": "One Click Away: AI Tutoring with Khanmigo in a Two-Year School Experiment", "status_or_publisher": "Annenberg Institute at Brown University (EdWorkingPapers)", "locator": "Abstract", "quote": "These gains resemble those from Khan Academy practice without AI assistance.", "source_url": "https://edworkingpapers.com/ai26-1551" }, { "note": "13", "kind": "study", "jurisdiction": "", "identifier": "", "title": "One Click Away: AI Tutoring with Khanmigo in a Two-Year School Experiment", "status_or_publisher": "Annenberg Institute at Brown University (EdWorkingPapers)", "locator": "Abstract", "quote": "96 percent of students tried Khanmigo at least once, but the median student messaged it on only a third of the days they practiced, and in only 17 percent of the exercise sessions in which they made a mistake.", "source_url": "https://edworkingpapers.com/ai26-1551" }, { "note": "14", "kind": "study", "jurisdiction": "", "identifier": "", "title": "Making AI Tutoring Productive: Evidence from a Mastery-Based Math Practice Experiment", "status_or_publisher": "National Bureau of Economic Research", "locator": "Abstract", "quote": "AI students progressed more slowly and attempted fewer questions, but answered more accurately conditional on reaching an attempt. The clearest mechanism appeared after mistakes: AI improved next-attempt correctness", "source_url": "https://www.nber.org/papers/w35621" }, { "note": "15", "kind": "study", "jurisdiction": "", "identifier": "", "title": "Making AI Tutoring Productive: Evidence from a Mastery-Based Math Practice Experiment", "status_or_publisher": "National Bureau of Economic Research", "locator": "Abstract", "quote": "The most encouraging delayed-test evidence appears when AI is embedded in the mastery workflow, with marginally significant gains concentrated on practiced Exercise 1 material.", "source_url": "https://www.nber.org/papers/w35621" }, { "note": "16", "kind": "study", "jurisdiction": "", "identifier": "", "title": "The Effects of Course-Integrated AI Tutoring on Student Performance and Engagement: A Randomized University Trial", "status_or_publisher": "Annenberg Institute at Brown University (EdWorkingPapers)", "locator": "Abstract", "quote": "Among sections of the same course, tutor access reduced final grades by 0.37 standard deviations and learning management system participation by 0.90 standard deviations.", "source_url": "https://edworkingpapers.com/ai26-1598" }, { "note": "17", "kind": "study", "jurisdiction": "", "identifier": "", "title": "The Effects of Course-Integrated AI Tutoring on Student Performance and Engagement: A Randomized University Trial", "status_or_publisher": "Annenberg Institute at Brown University (EdWorkingPapers)", "locator": "Introduction, p. 4", "quote": "the estimated final-grade losses among first-generation students were more than twice those among continuing-generation students in both samples", "source_url": "https://edworkingpapers.com/sites/default/files/ai26-1598.pdf" }, { "note": "18", "kind": "study", "jurisdiction": "", "identifier": "", "title": "Sustaining AI-Enabled Student Support: A Four-Year Implementation and Impact Study", "status_or_publisher": "Annenberg Institute at Brown University (EdWorkingPapers)", "locator": "Abstract", "quote": "Students remain receptive over time, and impacts are concentrated in improved completion of time-sensitive administrative tasks, with no detectable effects on academic performance or persistence.", "source_url": "https://edworkingpapers.com/ai26-1409" }, { "note": "19", "kind": "study", "jurisdiction": "", "identifier": "", "title": "ChatGPT-generated help produces learning gains equivalent to human tutor-authored help on mathematics skills", "status_or_publisher": "PLOS ONE", "locator": "Abstract", "quote": "We find that only the ChatGPT condition produces statistically significant learning gains compared to a no-help control, with no statistically significant differences in gains or time-on-task observed between learners receiving ChatGPT vs human tutor help.", "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11125466/" }, { "note": "20", "kind": "study", "jurisdiction": "", "identifier": "", "title": "ChatGPT-generated help produces learning gains equivalent to human tutor-authored help on mathematics skills", "status_or_publisher": "PLOS ONE", "locator": "Abstract", "quote": "Notably, ChatGPT-generated help failed quality checks on 32% of problems. This was, however, reducible to nearly 0% for algebra problems and 13% for statistics problems after applying self-consistency", "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11125466/" } ] }