<?xml version="1.0" encoding="utf-8"?>
<export-example>
  <doc>
    <id>2818</id>
    <completedYear>2024</completedYear>
    <publishedYear>2024</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst>31</pageFirst>
    <pageLast>37</pageLast>
    <pageNumber>7</pageNumber>
    <edition/>
    <issue/>
    <volume/>
    <type>conferenceobject</type>
    <publisherName>ACM</publisherName>
    <publisherPlace>New York, NY, USA</publisherPlace>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>1</belongsToBibliography>
    <completedDate>--</completedDate>
    <publishedDate>2024-07-03</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">Feedback-Generation for Programming Exercises With GPT-4</title>
    <abstract language="eng">Ever since Large Language Models (LLMs) and related applications have become broadly available, several studies investigated their potential for assisting educators and supporting students in higher education. LLMs such as Codex, GPT-3.5, and GPT 4 have shown promising results in the context of large programming courses, where students can benefit from feedback and hints if provided timely and at scale. This paper explores the quality of GPT-4 Turbo's generated output for prompts containing both the programming task specification and a student's submission as input. Two assignments from an introductory programming course were selected, and GPT-4 was asked to generate feedback for 55 randomly chosen, authentic student programming submissions. The output was qualitatively analyzed regarding correctness, personalization, fault localization, and other features identified in the material. Compared to prior work and analyses of GPT-3.5, GPT-4 Turbo shows notable improvements. For example, the output is more structured and consistent. GPT-4 Turbo can also accurately identify invalid casing in student programs' output. In some cases, the feedback also includes the output of the student program. At the same time, inconsistent feedback was noted such as stating that the submission is correct but an error needs to be fixed. The present work increases our understanding of LLMs' potential, limitations, and how to integrate them into e-assessment systems, pedagogical scenarios, and instructing students who are using applications based on GPT-4.</abstract>
    <parentTitle language="eng">Proceedings of the 2024 on Innovation and Technology in Computer Science Education Vol. 1</parentTitle>
    <identifier type="doi">10.1145/3649217.3653594</identifier>
    <enrichment key="opus_doi_flag">true</enrichment>
    <enrichment key="opus_import_data">{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2024,9,8]],"date-time":"2024-09-08T15:42:18Z","timestamp":1725810138523},"publisher-location":"New York, NY, USA","reference-count":43,"publisher":"ACM","funder":[{"name":"German Federal Ministry of Education and Research (BMBF)","award":["16DHBKI013"]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,3]]},"DOI":"10.1145\/3649217.3653594","type":"proceedings-article","created":{"date-parts":[[2024,7,3]],"date-time":"2024-07-03T22:30:20Z","timestamp":1720045820000},"page":"31-37","update-policy":"http:\/\/dx.doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["Feedback-Generation for Programming Exercises With GPT-4"],"prefix":"10.1145","author":[{"ORCID":"http:\/\/orcid.org\/0009-0005-6458-4169","authenticated-orcid":false,"given":"Imen","family":"Azaiz","sequence":"first","affiliation":[{"name":"LMU Munich, Munich, Germany"}]},{"ORCID":"http:\/\/orcid.org\/0000-0002-6843-2729","authenticated-orcid":false,"given":"Natalie","family":"Kiesler","sequence":"additional","affiliation":[{"name":"Nuremberg Tech, Nuremberg, Germany"}]},{"ORCID":"http:\/\/orcid.org\/0000-0002-9647-300X","authenticated-orcid":false,"given":"Sven","family":"Strickroth","sequence":"additional","affiliation":[{"name":"LMU Munich, Munich, Germany"}]}],"member":"320","published-online":{"date-parts":[[2024,7,3]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.3991\/ijep.v13i8.45621"},{"key":"e_1_3_2_1_2_1","unstructured":"Brett A Becker Michelle Craig Paul Denny Hieke Keuning Natalie Kiesler Juho Leinonen Andrew Luxton-Reilly James Prather and Keith Quille. 2023. Generative AI in Introductory Programming. (2023). https:\/\/csed.acm.org\/wp-content\/uploads\/2023\/12\/Generative-AI-Nov-2023-Version.pdf"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3344429.3372508"},{"key":"e_1_3_2_1_4_1","unstructured":"Douglas Bengtsson and Axel Kaliff. 2023. Assessment Accuracy of a Large Language Model on Programming Assignments. https:\/\/urn.kb.se\/resolve?urn=urn:nbn:se:kth:diva-331000"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1191\/1478088706qp063oa"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2306.02608"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.2190\/3LFX-9RRF-67T8-UVK9"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/EDUCON.2016.7474580"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3511861.3511863"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3576123.3576134"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1080\/08993408.2020.1860408"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3571785.3574124"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3408877.3432526"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3231711"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341525.3387419"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1007\/978--3-031--47148--3"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/FIE58773.2023.10343457"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","unstructured":"Natalie Kiesler and Daniel Schiffner. 2023. Large Language Models in Introductory Programming Education: ChatGPT's Performance and Implications for Assessments. In CoRR abs\/2308.08572. arXiv: 2308.08572. https:\/\/doi.org\/10.48550\/arXiv.2308.08572 arxiv: 2308.08572 [cs.SE]","DOI":"10.48550\/arXiv.2308.08572"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","unstructured":"Nguyen-Thinh Le Sven Strickroth Sebastian Gross and Niels Pinkwart. 2013. A Review of AI-Supported Tutoring Approaches for Learning Programming. In Advanced Computational Methods for Knowledge Engineering - Proceedings of the 1st International Conference on Computer Science Applied Mathematics and Applications (ICCSAMA) (Studies in Computational Intelligence 479). Springer Verlag Berlin Germany 267--279. https:\/\/doi.org\/10.1007\/978--3--319-00293--4_20","DOI":"10.1007\/978--3--319-00293--4_20"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3587102.3588785"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569770"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3385209.3385226"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/2899415.2899432"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293881.3295779"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569785"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.17169\/FQS-2.1.967"},{"volume-title":"Feedback strategies for interactive learning tasks. Handbook of research on educational communications and technology","year":"2008","author":"Narciss Susanne","key":"e_1_3_2_1_27_1","unstructured":"Susanne Narciss. 2008. Feedback strategies for interactive learning tasks. Handbook of research on educational communications and technology, Vol. 3 (2008), 125--144."},{"key":"e_1_3_2_1_28_1","unstructured":"OpenAI. [n. d.]. GPT-4 Turbo. https:\/\/help.openai.com\/en\/articles\/8555510-gpt-4-turbo"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2999541.2999552"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3623762.3633499"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"crossref","unstructured":"Lianne Roest Hieke Keuning and Johan Jeuring. 2023. Next-Step Hint Generation for Introductory Programming Using Large Language Models. arxiv: 2312.10055 [cs.CY]","DOI":"10.1145\/3636243.3636259"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3501385.3543957"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"crossref","unstructured":"Jaromir Savelka Arav Agarwal Christopher Bogart and Majd Sakr. 2023. Large Language Models (GPT) Struggle to Answer Multiple-Choice Questions about Code. arxiv: 2303.08033 [cs.CL]","DOI":"10.5220\/0011996900003470"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.3102\/0034654307313795"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/6138.6145"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3587102.3588802"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.5220\/0011087700003182"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1007\/978--3-031--20617--7_3"},{"volume-title":"Proc. DeLFI. Gesellschaft f\u00fcr Informatik e.V., Bonn, 115--126","year":"2011","author":"Strickroth Sven","key":"e_1_3_2_1_39_1","unstructured":"Sven Strickroth, Hannes Olivier, and Niels Pinkwart. 2011. Das GATE-System: Qualit\u00e4tssteigerung durch Selbsttests f\u00fcr Studenten bei der Onlineabgabe von \u00dcbungsaufgaben?. In Proc. DeLFI. Gesellschaft f\u00fcr Informatik e.V., Bonn, 115--126. https:\/\/dl.gi.de\/handle\/20.500.12116\/4740"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.3991\/ijep.v12i5.31283"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1016\/0959--4752(94)90003--5"},{"volume-title":"The many ways of the Bracelet project. BACIT","year":"2007","author":"Whalley Jacqueline","key":"e_1_3_2_1_42_1","unstructured":"Jacqueline Whalley, Tony Clear, and Raymond Lister. 2007. The many ways of the Bracelet project. BACIT (2007)."},{"key":"e_1_3_2_1_43_1","unstructured":"Yonghao Wu Zheng Li Jie M. Zhang Mike Papadakis Mark Harman and Yong Liu. 2023. Large Language Models in Fault Localisation. arxiv: 2308.15276 [cs.SE]"}],"event":{"name":"ITiCSE 2024: Innovation and Technology in Computer Science Education","sponsor":["SIGCSE ACM Special Interest Group on Computer Science Education"],"location":"Milan Italy","acronym":"ITiCSE 2024"},"container-title":["Proceedings of the 2024 on Innovation and Technology in Computer Science Education V. 1"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3649217.3653594","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,8,3]],"date-time":"2024-08-03T10:58:24Z","timestamp":1722682704000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3649217.3653594"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,3]]},"references-count":43,"alternative-id":["10.1145\/3649217.3653594","10.1145\/3649217"],"URL":"https:\/\/doi.org\/10.1145\/3649217.3653594","relation":{},"subject":[],"published":{"date-parts":[[2024,7,3]]},"assertion":[{"value":"2024-07-03","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}</enrichment>
    <enrichment key="local_crossrefDocumentType">proceedings-article</enrichment>
    <enrichment key="local_import_origin">crossref</enrichment>
    <enrichment key="local_doiImportPopulated">PersonAuthorFirstName_1,PersonAuthorLastName_1,PersonAuthorIdentifierOrcid_1,PersonAuthorFirstName_2,PersonAuthorLastName_2,PersonAuthorIdentifierOrcid_2,PersonAuthorFirstName_3,PersonAuthorLastName_3,PersonAuthorIdentifierOrcid_3,Enrichmentconference_title,Enrichmentconference_place,PublisherName,PublisherPlace,TitleMain_1,TitleParent_1,PageNumber,PageFirst,PageLast,CompletedYear</enrichment>
    <enrichment key="conference_title">ITiCSE 2024: Innovation and Technology in Computer Science Education</enrichment>
    <enrichment key="conference_place">Milan Italy</enrichment>
    <enrichment key="Reviewstatus">Begutachtet/Reviewed</enrichment>
    <enrichment key="opus.source">doi-import</enrichment>
    <enrichment key="opus.doi.autoCreate">false</enrichment>
    <enrichment key="opus.urn.autoCreate">false</enrichment>
    <author>Imen Azaiz</author>
    <author>Natalie Kiesler</author>
    <author>Sven Strickroth</author>
    <collection role="institutes" number="">Fakultät Informatik</collection>
    <collection role="Forschungsschwerpunkt" number="5">Digitalisierung &amp; Künstliche Intelligenz</collection>
  </doc>
</export-example>
