@inproceedings{StrussRuggeriBarronCedenoetal., author = {Struß, Julia Maria and Ruggeri, Federico and Barr{\´o}n-Cede{\~n}o, Alberto and Alam, Firoj and Dimitrov, Dimitar and Galassi, Andrea and Pachov, Georgi and Koychev, Ivan and Nakov, Preslav and Siegel, Melanie and Wiegand, Michael and Hasanain, Maram and Suwaileh, Reem and Zaghouani, Wajdi}, title = {Overview of the CLEF-2024 CheckThat! Lab Task 2 on Subjectivity in News Articles}, series = {CLEF 2024 Working Notes : Working Notes of the Conference and Labs of the Evaluation Forum (CLEF 2024)}, booktitle = {CLEF 2024 Working Notes : Working Notes of the Conference and Labs of the Evaluation Forum (CLEF 2024)}, editor = {Faggioli, Guglielmo and Ferro, Nicola and Galušč{\´a}kov{\´a}, Petra and Seco de Herrera, Alba Garc{\´i}a}, address = {Frankreich}, organization = {University of Grenoble Alpes}, issn = {1613-0073}, url = {http://nbn-resolving.de/urn:nbn:de:0074-3740-3}, pages = {287 -- 298}, abstract = {We present an overview of Task 2 of the seventh edition of the CheckThat! lab at the 2024 iteration of the Conference and Labs of the Evaluation Forum (CLEF). The task focuses on subjectivity detection in news articles and was o ered in five languages: Arabic, Bulgarian, English, German, and Italian, as well as in a multilingual setting. The datasets for each language were carefully curated and annotated, comprising over 10,000 sentences from news articles. The task challenged participants to develop systems capable of distinguishing between subjective statements (refecting personal opinions or biases) and objective ones (presenting factual information) at the sentence level. A total of 15 teams participated in the task, submitting 36 valid runs across all language tracks. The participants used a variety of approaches, with transformer-based models being the most popular choice. Strategies included fine-tuning monolingual and multilingual models, and leveraging English models with automatic translation for the non-English datasets. Some teams also explored ensembles, feature engineering, and innovative techniques such as few-shot learning and in-context learning with large language models. The evaluation was based on macro-averaged F1 score. The results varied across languages, with the best performance achieved for Italian and German, followed by English. The Arabic track proved particularly challenging, with no team surpassing an F1 score of 0.50. This task contributes to the broader goal of enhancing the reliability of automated content analysis in the context of misinformation detection and fact-checking. The paper provides detailed insights into the datasets, participant approaches, and results, o ering a benchmark for the current state of subjectivity detection across multiple languages.}, subject = {Fehlinformation}, language = {en} } @inproceedings{BarronCedenoAlamCasellietal., author = {Barr{\´o}n-Cede{\~n}o, Alberto and Alam, Firoj and Caselli, Tommaso and Da San Martino, Giovanni and Elsayed, Tamer and Galassi, Andrea and Haouari, Fatima and Ruggeri, Federico and Struß, Julia Maria and Nath Nandi, Rabindra and Cheema, Gullal S. and Azizov, Dilshod and Nakov, Preslav}, title = {The CLEF-2023 CheckThat! Lab: Checkworthiness, Subjectivity, Political Bias, Factuality, and Authority}, series = {Advances in Information Retrieval : 45th European Conference on Information Retrieval, ECIR 2023, Dublin, Ireland, April 2-6, 2023, Proceedings, Part III}, booktitle = {Advances in Information Retrieval : 45th European Conference on Information Retrieval, ECIR 2023, Dublin, Ireland, April 2-6, 2023, Proceedings, Part III}, publisher = {Springer}, address = {Cham}, isbn = {978-3-031-28241-6}, issn = {0302-9743}, doi = {10.1007/978-3-031-28241-6_59}, pages = {509 -- 517}, abstract = {The five editions of the CheckThat! lab so far have focused on the main tasks of the information verification pipeline: check-worthiness, evidence retrieval and pairing, and verification. The 2023 edition of the lab zooms into some of the problems and - for the first time - it offers five tasks in seven languages (Arabic, Dutch, English, German, Italian, Spanish, and Turkish): Task 1 asks to determine whether an item, text or a text plus an image, is check-worthy; Task 2 requires to assess whether a text snippet is subjective or not; Task 3 looks for estimating the political bias of a document or a news outlet; Task 4 requires to determine the level of factuality of a document or a news outlet; and Task 5 is about identifying authorities that should be trusted to verify a contended claim.}, subject = {Desinformation}, language = {en} } @inproceedings{BarronCedenoAlamGalassietal., author = {Barr{\´o}n-Cede{\~n}o, Alberto and Alam, Firoj and Galassi, Andrea and Da San Martino, Giovanni and Nakov, Preslav and Elsayed, Tamer and Azizov, Dilshod and Caselli, Tommaso and Cheema, Gullal S. and Haouari, Fatima and Hasanain, Maram and Kutlu, Mucahid and Li, Chengkai and Ruggeri, Federico and Struß, Julia Maria and Zaghouani, Wajdi}, title = {Overview of the CLEF-2023 CheckThat! Lab on Checkworthiness, Subjectivity, Political Bias, Factuality, and Authority of News Articles and Their Source}, series = {Experimental IR Meets Multilinguality, Multimodality, and Interaction}, booktitle = {Experimental IR Meets Multilinguality, Multimodality, and Interaction}, editor = {Arampatzis, Avi and Kanoulas, Evangelos and Tsikrika, Theodora and Vrochidis, Stefanos and Giachanou, Anastasia and Li, Dan and Aliannejadi, Mohammad and Vlachos, Michalis and Faggioli, Guglielmo and Ferro, Nicola}, publisher = {Springer International Publishing}, address = {Cham}, isbn = {978-3-031-42448-9}, issn = {1611-3349}, doi = {10.1007/978-3-031-42448-9_20}, pages = {251 -- 275}, abstract = {We describe the sixth edition of the CheckThat! lab, part of the 2023 Conference and Labs of the Evaluation Forum (CLEF). The five previous editions of CheckThat! focused on the main tasks of the information verification pipeline: check-worthiness, verifying whether a claim was fact-checked before, supporting evidence retrieval, and claim verification. In this sixth edition, we zoom into some new problems and for the first time we offer five tasks in seven languages: Arabic, Dutch, English, German, Italian, Spanish, and Turkish. Task 1 asks to determine whether an item - text or text plus image- is check-worthy. Task 2 aims to predict whether a sentence from a news article is subjective or not. Task 3 asks to assess the political bias of the news at the article and at the media outlet level. Task 4 focuses on the factuality of reporting of news media. Finally, Task 5 looks at identifying authorities in Twitter that could help verify a given target claim. For a second year, CheckThat! was the most popular lab at CLEF-2023 in terms of team registrations: 127 teams. About one-third of them (a total of 37) actually participated.}, subject = {Desinformation}, language = {en} } @inproceedings{GalassiRuggeriBarronCedenoetal., author = {Galassi, Andrea and Ruggeri, Federico and Barr{\´o}n-Cede{\~n}o, Alberto and Alam, Firoj and Caselli, Tommaso and Kutlu, Mucahid and Struß, Julia Maria and Antici, Francesco and Hasanain, Maram and K{\"o}hler, Juliane and Korre, Katerina and Leistra, Folkert and Muti, Arianna and Siegel, Melanie and T{\"u}rkmen, Mehmet Deniz and Wiegand, Michael and Zaghouani, Wajdi}, title = {Overview of the CLEF-2023 CheckThat! Lab: Task 2 on Subjectivity in News Articles}, series = {CLEF 2023 Working Notes}, booktitle = {CLEF 2023 Working Notes}, editor = {Aliannejadi, Mohammad and Faggiolo, Guglielmo and Ferro, Nicola and Vlachos, Michalis}, address = {Thessaloniki}, organization = {Centre for Research and Technology Hellas}, pages = {236 -- 249}, abstract = {We describe the outcome of the 2023 edition of the CheckThat!Lab at CLEF. We focus on subjectivity (Task 2), which has been proposed for the first time. It aims at fostering the technology for the identification of subjective text fragments in news articles. For that, we produced corpora consisting of 9,530 manually-annotated sentences, covering six languages - Arabic, Dutch, English, German, Italian, and Turkish. Task 2 attracted 12 teams, which submitted a total of 40 final runs covering all languages. The most successful approaches addressed the task using state-of-the-art multilingual transformer models, which were fine-tuned on language-specific data. Teams also experimented with a rich set of other neural architectures, including foundation models, zero-shot classifiers, and standard transformers, mainly coupled with data augmentation and multilingual training strategies to address class imbalance. We publicly release all the datasets and evaluation scripts, with the purpose of promoting further research on this topic.}, subject = {Desinformation}, language = {en} }