{"id":"https://openalex.org/W4389260805","doi":"https://doi.org/10.48550/arxiv.2311.18140","title":"ROBBIE: Robust Bias Evaluation of Large Generative Language Models","display_name":"ROBBIE: Robust Bias Evaluation of Large Generative Language Models","publication_year":2023,"publication_date":"2023-11-29","ids":{"openalex":"https://openalex.org/W4389260805","doi":"https://doi.org/10.48550/arxiv.2311.18140"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2311.18140","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2311.18140","pdf_url":"https://arxiv.org/pdf/2311.18140","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2311.18140","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5060413371","display_name":"David Esiobu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Esiobu, David","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109301242","display_name":"Xiaoqing Tan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tan, Xiaoqing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080012348","display_name":"Saghar Hosseini","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hosseini, Saghar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067335278","display_name":"Megan Ung","orcid":"https://orcid.org/0000-0003-3063-3207"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ung, Megan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100424305","display_name":"Yuchen Zhang","orcid":"https://orcid.org/0000-0001-8536-7175"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yuchen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026363324","display_name":"Jude Fernandes","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fernandes, Jude","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060784472","display_name":"Jane Dwivedi-Yu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dwivedi-Yu, Jane","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036470617","display_name":"E. Presani","orcid":"https://orcid.org/0000-0001-6254-9603"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Presani, Eleonora","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062696000","display_name":"Adina Williams","orcid":"https://orcid.org/0000-0001-5281-3343"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Williams, Adina","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5007334392","display_name":"Eric M. Smith","orcid":"https://orcid.org/0000-0002-5385-4558"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Smith, Eric Michael","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.974399983882904,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.974399983882904,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.9129999876022339,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.7246102094650269},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.5908043384552002},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.4858986735343933},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4781794846057892},{"id":"https://openalex.org/keywords/meaning","display_name":"Meaning (existential)","score":0.4297041893005371},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3849647641181946},{"id":"https://openalex.org/keywords/econometrics","display_name":"Econometrics","score":0.32952654361724854},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.323292076587677},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.28727003931999207},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.2802848219871521},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.21726828813552856},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.13575339317321777},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.12279012799263},{"id":"https://openalex.org/keywords/marketing","display_name":"Marketing","score":0.11276400089263916}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.7246102094650269},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.5908043384552002},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.4858986735343933},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4781794846057892},{"id":"https://openalex.org/C2780876879","wikidata":"https://www.wikidata.org/wiki/Q3054749","display_name":"Meaning (existential)","level":2,"score":0.4297041893005371},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3849647641181946},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.32952654361724854},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.323292076587677},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.28727003931999207},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2802848219871521},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.21726828813552856},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.13575339317321777},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.12279012799263},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.11276400089263916},{"id":"https://openalex.org/C542102704","wikidata":"https://www.wikidata.org/wiki/Q183257","display_name":"Psychotherapist","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2311.18140","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2311.18140","pdf_url":"https://arxiv.org/pdf/2311.18140","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2311.18140","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2311.18140","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2311.18140","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2311.18140","pdf_url":"https://arxiv.org/pdf/2311.18140","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10","score":0.49000000953674316}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4389260805.pdf","grobid_xml":"https://content.openalex.org/works/W4389260805.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4238897586","https://openalex.org/W435179959","https://openalex.org/W2619091065","https://openalex.org/W2059640416","https://openalex.org/W1490753184","https://openalex.org/W2284465472","https://openalex.org/W2291782699","https://openalex.org/W1993948687","https://openalex.org/W2000169967","https://openalex.org/W2112883198"],"abstract_inverted_index":{"As":[0],"generative":[1,94],"large":[2],"language":[3],"models":[4],"(LLMs)":[5],"grow":[6],"more":[7,45,54],"performant":[8],"and":[9,19,37,56,60,83,90,102,122,141,203],"prevalent,":[10],"we":[11,129,151],"must":[12],"develop":[13],"comprehensive":[14,154],"enough":[15],"tools":[16],"to":[17,29,146,171,183],"measure":[18,30,186],"improve":[20],"their":[21,52],"fairness.":[22],"Different":[23],"prompt-based":[24,81],"datasets":[25,46,106],"can":[26,47],"be":[27],"used":[28],"social":[31],"bias":[32,82,121,219],"across":[33,86,164],"multiple":[34],"text":[35],"domains":[36],"demographic":[38,65,88,134],"axes,":[39],"meaning":[40],"that":[41],"testing":[42],"LLMs":[43],"on":[44],"potentially":[48],"help":[49],"us":[50,117],"characterize":[51],"biases":[53],"fully,":[55],"better":[57],"ensure":[58],"equal":[59],"equitable":[61],"treatment":[62],"of":[63,78,93,97,113,124,133,156,167,214,218],"marginalized":[64],"groups.":[66],"In":[67],"this":[68,143],"work,":[69],"our":[70,165,209],"focus":[71],"is":[72],"two-fold:":[73],"(1)":[74],"Benchmarking:":[75],"a":[76,153,178],"comparison":[77,112],"6":[79,99],"different":[80],"toxicity":[84,123],"metrics":[85],"12":[87],"axes":[89],"5":[91],"families":[92],"LLMs.":[95,222],"Out":[96],"those":[98,114],"metrics,":[100],"AdvPromptSet":[101],"HolisticBiasR":[103],"are":[104],"novel":[105],"proposed":[107],"in":[108,136,212,220],"the":[109,120,125,131,181,197],"paper.":[110],"The":[111],"benchmarks":[115],"gives":[116],"insights":[118,173],"about":[119],"compared":[126],"models.":[127],"Therefore,":[128],"explore":[130],"frequency":[132],"terms":[135],"common":[137],"LLM":[138],"pre-training":[139],"corpora":[140],"how":[142,157,192],"may":[144],"relate":[145],"model":[147],"biases.":[148],"(2)":[149],"Mitigation:":[150],"conduct":[152],"study":[155],"well":[158],"3":[159],"bias/toxicity":[160],"mitigation":[161],"techniques":[162],"perform":[163],"suite":[166],"measurements.":[168],"ROBBIE":[169],"aims":[170],"provide":[172],"for":[174],"practitioners":[175],"while":[176],"deploying":[177],"model,":[179],"emphasizing":[180],"need":[182],"not":[184],"only":[185],"potential":[187],"harms,":[188],"but":[189],"also":[190],"understand":[191],"they":[193],"arise":[194],"by":[195],"characterizing":[196],"data,":[198],"mitigate":[199],"harms":[200],"once":[201],"found,":[202],"balance":[204],"any":[205],"trade-offs.":[206],"We":[207],"open-source":[208],"analysis":[210],"code":[211],"hopes":[213],"encouraging":[215],"broader":[216],"measurements":[217],"future":[221]},"counts_by_year":[{"year":2024,"cited_by_count":2}],"updated_date":"2026-08-18T07:49:30.821534","created_date":"2025-10-10T00:00:00"}