@inproceedings{NakovDaSanMartinoElsayedetal., author = {Nakov, Preslav and Da San Martino, Giovanni and Elsayed, Tamer and Barr{\´o}n-Cede{\~n}o, Alberto and M{\´i}guez, Rub{\´e}n and Shaar, Shaden and Alam, Firoj and Haouari, Fatima and Hasanain, Maram and Mansour, Watheq and Hamdan, Bayan and Sheikh Ali, Zien and Babulkov, Nikolay and Nikolov, Alex and Koshore Shahi, Gautam and Struß, Julia Maria and Mandl, Thomas and Kutlu, Mucahid and Selim Kartal, Yavuz}, title = {Overview of the CLEF-2021 CheckThat! Lab on Detecting Check-Worthy Claims, Previously Fact-Checked Claims, and Fake News}, series = {Experimental IR Meets Multilinguality, Multimodality, and Interaction}, booktitle = {Experimental IR Meets Multilinguality, Multimodality, and Interaction}, publisher = {Springer}, address = {Cham}, isbn = {978-3-030-85251-1}, doi = {10.1007/978-3-030-85251-1_19}, pages = {264 -- 291}, abstract = {We describe the fourth edition of the CheckThat! Lab, part of the 2021 Conference and Labs of the Evaluation Forum (CLEF). The lab evaluates technology supporting tasks related to factuality, and covers Arabic, Bulgarian, English, Spanish, and Turkish. Task 1 asks to predict which posts in a Twitter stream are worth fact-checking, focusing on COVID-19 and politics (in all five languages). Task 2 asks to determine whether a claim in a tweet can be verified using a set of previously fact-checked claims (in Arabic and English). Task 3 asks to predict the veracity of a news article and its topical domain (in English). The evaluation is based on mean average precision or precision at rank k for the ranking tasks, and macro-F1 for the classification tasks. This was the most popular CLEF-2021 lab in terms of team registrations: 132 teams. Nearly one-third of them participated: 15, 5, and 25 teams submitted official runs for tasks 1, 2, and 3, respectively.}, subject = {Desinformation}, language = {en} } @inproceedings{NakovDaSanMartinoElsayedetal., author = {Nakov, Preslav and Da San Martino, Giovanni and Elsayed, Tamer and Barr{\´o}n-Cede{\~n}o, Alberto and M{\´i}guez, R{\´u}ben and Shaar, Shaden and Alam, Firoj and Haouari, Fatima and Hasanain, Maram and Babulkov, Nikolay and Nikolov, Alex and Shahi, Gautam Kishore and Struß, Julia Maria and Mandl, Thomas}, title = {The CLEF-2021 CheckThat! Lab on Detecting Check-Worthy Claims, Previously Fact-Checked Claims, and Fake News}, series = {Advances in Information Retrieval}, booktitle = {Advances in Information Retrieval}, publisher = {Springer}, address = {Cham}, isbn = {978-3-030-72240-1}, doi = {10.1007/978-3-030-72240-1_75}, pages = {639 -- 649}, abstract = {We describe the fourth edition of the CheckThat! Lab, part of the 2021 Cross-Language Evaluation Forum (CLEF). The lab evaluates technology supporting various tasks related to factuality, and it is offered in Arabic, Bulgarian, English, and Spanish. Task 1 asks to predict which tweets in a Twitter stream are worth fact-checking (focusing on COVID-19). Task 2 asks to determine whether a claim in a tweet can be verified using a set of previously fact-checked claims. Task 3 asks to predict the veracity of a target news article and its topical domain. The evaluation is carried out using mean average precision or precision at rank k for the ranking tasks, and F1 for the classification tasks.}, subject = {Falschmeldung}, language = {en} } @inproceedings{BarronCedenoAlamGalassietal., author = {Barr{\´o}n-Cede{\~n}o, Alberto and Alam, Firoj and Galassi, Andrea and Da San Martino, Giovanni and Nakov, Preslav and Elsayed, Tamer and Azizov, Dilshod and Caselli, Tommaso and Cheema, Gullal S. and Haouari, Fatima and Hasanain, Maram and Kutlu, Mucahid and Li, Chengkai and Ruggeri, Federico and Struß, Julia Maria and Zaghouani, Wajdi}, title = {Overview of the CLEF-2023 CheckThat! Lab on Checkworthiness, Subjectivity, Political Bias, Factuality, and Authority of News Articles and Their Source}, series = {Experimental IR Meets Multilinguality, Multimodality, and Interaction}, booktitle = {Experimental IR Meets Multilinguality, Multimodality, and Interaction}, editor = {Arampatzis, Avi and Kanoulas, Evangelos and Tsikrika, Theodora and Vrochidis, Stefanos and Giachanou, Anastasia and Li, Dan and Aliannejadi, Mohammad and Vlachos, Michalis and Faggioli, Guglielmo and Ferro, Nicola}, publisher = {Springer International Publishing}, address = {Cham}, isbn = {978-3-031-42448-9}, issn = {1611-3349}, doi = {10.1007/978-3-031-42448-9_20}, pages = {251 -- 275}, abstract = {We describe the sixth edition of the CheckThat! lab, part of the 2023 Conference and Labs of the Evaluation Forum (CLEF). The five previous editions of CheckThat! focused on the main tasks of the information verification pipeline: check-worthiness, verifying whether a claim was fact-checked before, supporting evidence retrieval, and claim verification. In this sixth edition, we zoom into some new problems and for the first time we offer five tasks in seven languages: Arabic, Dutch, English, German, Italian, Spanish, and Turkish. Task 1 asks to determine whether an item - text or text plus image- is check-worthy. Task 2 aims to predict whether a sentence from a news article is subjective or not. Task 3 asks to assess the political bias of the news at the article and at the media outlet level. Task 4 focuses on the factuality of reporting of news media. Finally, Task 5 looks at identifying authorities in Twitter that could help verify a given target claim. For a second year, CheckThat! was the most popular lab at CLEF-2023 in terms of team registrations: 127 teams. About one-third of them (a total of 37) actually participated.}, subject = {Desinformation}, language = {en} } @inproceedings{GalassiRuggeriBarronCedenoetal., author = {Galassi, Andrea and Ruggeri, Federico and Barr{\´o}n-Cede{\~n}o, Alberto and Alam, Firoj and Caselli, Tommaso and Kutlu, Mucahid and Struß, Julia Maria and Antici, Francesco and Hasanain, Maram and K{\"o}hler, Juliane and Korre, Katerina and Leistra, Folkert and Muti, Arianna and Siegel, Melanie and T{\"u}rkmen, Mehmet Deniz and Wiegand, Michael and Zaghouani, Wajdi}, title = {Overview of the CLEF-2023 CheckThat! Lab: Task 2 on Subjectivity in News Articles}, series = {CLEF 2023 Working Notes}, booktitle = {CLEF 2023 Working Notes}, editor = {Aliannejadi, Mohammad and Faggiolo, Guglielmo and Ferro, Nicola and Vlachos, Michalis}, address = {Thessaloniki}, organization = {Centre for Research and Technology Hellas}, pages = {236 -- 249}, abstract = {We describe the outcome of the 2023 edition of the CheckThat!Lab at CLEF. We focus on subjectivity (Task 2), which has been proposed for the first time. It aims at fostering the technology for the identification of subjective text fragments in news articles. For that, we produced corpora consisting of 9,530 manually-annotated sentences, covering six languages - Arabic, Dutch, English, German, Italian, and Turkish. Task 2 attracted 12 teams, which submitted a total of 40 final runs covering all languages. The most successful approaches addressed the task using state-of-the-art multilingual transformer models, which were fine-tuned on language-specific data. Teams also experimented with a rich set of other neural architectures, including foundation models, zero-shot classifiers, and standard transformers, mainly coupled with data augmentation and multilingual training strategies to address class imbalance. We publicly release all the datasets and evaluation scripts, with the purpose of promoting further research on this topic.}, subject = {Desinformation}, language = {en} } @inproceedings{BarronCedenoAlamChakrabortyetal., author = {Barr{\´o}n-Cede{\~n}o, Alberto and Alam, Firoj and Chakraborty, Tonmoy and Elsayed, Tamer and Nakov, Preslav and Przybyła, Piotr and Struß, Julia Maria and Haouari, Fatima and Hasanain, Maram and Ruggeri, Federico and Song, Xingyi and Suwaileh, Reem}, title = {The CLEF-2024 CheckThat! Lab}, series = {Advances in Information Retrieval}, booktitle = {Advances in Information Retrieval}, editor = {Goharian, Nazli and Tonellotto, Nicola and He, Yulan and Lipani, Aldo and McDonald, Graham and Macdonald, Craig and Ounis, Iadh}, publisher = {Springer International Publishing}, address = {Cham}, isbn = {978-3-031-56069-9}, issn = {0302-9743}, doi = {10.1007/978-3-031-56069-9_62}, pages = {449 -- 458}, abstract = {The first five editions of the CheckThat! lab focused on the main tasks of the information verification pipeline: check-worthiness, evidence retrieval and pairing, and verification. Since the 2023 edition, it has been focusing on new problems that can support the research and decision making during the verification process. In this new edition, we focus on new problems and -for the first time- we propose six tasks in fifteen languages (Arabic, Bulgarian, English, Dutch, French, Georgian, German, Greek, Italian, Polish, Portuguese, Russian, Slovene, Spanish, and code-mixed Hindi-English): Task 1 estimation of check-worthiness (the only task that has been present in all CheckThat! editions), Task 2 identification of subjectivity (a follow up of CheckThat! 2023 edition), Task 3 identification of persuasion (a follow up of SemEval 2023), Task 4 detection of hero, villain, and victim from memes (a follow up of CONSTRAINT 2022), Task 5 Rumor Verification using Evidence from Authorities (a first), and Task 6 robustness of credibility assessment with adversarial examples (a first). These tasks represent challenging classification and retrieval problems at the document and at the span level, including multilingual and multimodal settings.}, subject = {Desinformation}, language = {en} }