@inproceedings{pikuliak-etal-2023-depth,
title = "In-Depth Look at Word Filling Societal Bias Measures",
author = "Pikuliak, Mat{\'u}{\v{s}} and
Be{\v{n}}ov{\'a}, Ivana and
Bachrat{\'y}, Viktor",
editor = "Vlachos, Andreas and
Augenstein, Isabelle",
booktitle = "Proceedings of the 17th Conference of the European Chapter of the Association for Computational Linguistics",
month = may,
year = "2023",
address = "Dubrovnik, Croatia",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2023.eacl-main.265/",
doi = "10.18653/v1/2023.eacl-main.265",
pages = "3648--3665",
abstract = "Many measures of societal bias in language models have been proposed in recent years. A popular approach is to use a set of word filling prompts to evaluate the behavior of the language models. In this work, we analyze the validity of two such measures {--} StereoSet and CrowS-Pairs. We show that these measures produce unexpected and illogical results when appropriate control group samples are constructed. Based on this, we believe that they are problematic and using them in the future should be reconsidered. We propose a way forward with an improved testing protocol. Finally, we also introduce a new gender bias dataset for Slovak."
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="pikuliak-etal-2023-depth">
<titleInfo>
<title>In-Depth Look at Word Filling Societal Bias Measures</title>
</titleInfo>
<name type="personal">
<namePart type="given">Matúš</namePart>
<namePart type="family">Pikuliak</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ivana</namePart>
<namePart type="family">Beňová</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Viktor</namePart>
<namePart type="family">Bachratý</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2023-05</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 17th Conference of the European Chapter of the Association for Computational Linguistics</title>
</titleInfo>
<name type="personal">
<namePart type="given">Andreas</namePart>
<namePart type="family">Vlachos</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Isabelle</namePart>
<namePart type="family">Augenstein</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Dubrovnik, Croatia</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Many measures of societal bias in language models have been proposed in recent years. A popular approach is to use a set of word filling prompts to evaluate the behavior of the language models. In this work, we analyze the validity of two such measures – StereoSet and CrowS-Pairs. We show that these measures produce unexpected and illogical results when appropriate control group samples are constructed. Based on this, we believe that they are problematic and using them in the future should be reconsidered. We propose a way forward with an improved testing protocol. Finally, we also introduce a new gender bias dataset for Slovak.</abstract>
<identifier type="citekey">pikuliak-etal-2023-depth</identifier>
<identifier type="doi">10.18653/v1/2023.eacl-main.265</identifier>
<location>
<url>https://aclanthology.org/2023.eacl-main.265/</url>
</location>
<part>
<date>2023-05</date>
<extent unit="page">
<start>3648</start>
<end>3665</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T In-Depth Look at Word Filling Societal Bias Measures
%A Pikuliak, Matúš
%A Beňová, Ivana
%A Bachratý, Viktor
%Y Vlachos, Andreas
%Y Augenstein, Isabelle
%S Proceedings of the 17th Conference of the European Chapter of the Association for Computational Linguistics
%D 2023
%8 May
%I Association for Computational Linguistics
%C Dubrovnik, Croatia
%F pikuliak-etal-2023-depth
%X Many measures of societal bias in language models have been proposed in recent years. A popular approach is to use a set of word filling prompts to evaluate the behavior of the language models. In this work, we analyze the validity of two such measures – StereoSet and CrowS-Pairs. We show that these measures produce unexpected and illogical results when appropriate control group samples are constructed. Based on this, we believe that they are problematic and using them in the future should be reconsidered. We propose a way forward with an improved testing protocol. Finally, we also introduce a new gender bias dataset for Slovak.
%R 10.18653/v1/2023.eacl-main.265
%U https://aclanthology.org/2023.eacl-main.265/
%U https://doi.org/10.18653/v1/2023.eacl-main.265
%P 3648-3665
Markdown (Informal)
[In-Depth Look at Word Filling Societal Bias Measures](https://aclanthology.org/2023.eacl-main.265/) (Pikuliak et al., EACL 2023)
ACL
- Matúš Pikuliak, Ivana Beňová, and Viktor Bachratý. 2023. In-Depth Look at Word Filling Societal Bias Measures. In Proceedings of the 17th Conference of the European Chapter of the Association for Computational Linguistics, pages 3648–3665, Dubrovnik, Croatia. Association for Computational Linguistics.