{"id":"https://openalex.org/W4313483387","doi":"https://doi.org/10.48550/arxiv.2301.00068","title":"Inconsistencies in Masked Language Models","display_name":"Inconsistencies in Masked Language Models","publication_year":2022,"publication_date":"2022-12-30","ids":{"openalex":"https://openalex.org/W4313483387","doi":"https://doi.org/10.48550/arxiv.2301.00068"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2301.00068","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2301.00068","pdf_url":"https://arxiv.org/pdf/2301.00068","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2301.00068","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5112881223","display_name":"Tom Young","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Young, Tom","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Chen, Yunan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Yunan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"You, Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"You, Yang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9771999716758728,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.704537034034729},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6885837316513062},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6564638614654541},{"id":"https://openalex.org/keywords/masking","display_name":"Masking (illustration)","score":0.6044214963912964},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.5093742609024048},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5032441020011902},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5016796588897705},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.4909496605396271},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.45309075713157654},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.35236477851867676},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.05863600969314575}],"concepts":[{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.704537034034729},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6885837316513062},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6564638614654541},{"id":"https://openalex.org/C2777402240","wikidata":"https://www.wikidata.org/wiki/Q6783436","display_name":"Masking (illustration)","level":2,"score":0.6044214963912964},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.5093742609024048},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5032441020011902},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5016796588897705},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.4909496605396271},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.45309075713157654},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.35236477851867676},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.05863600969314575},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.0},{"id":"https://openalex.org/C146978453","wikidata":"https://www.wikidata.org/wiki/Q3798668","display_name":"Aerospace engineering","level":1,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2301.00068","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2301.00068","pdf_url":"https://arxiv.org/pdf/2301.00068","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2301.00068","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2301.00068","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2301.00068","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2301.00068","pdf_url":"https://arxiv.org/pdf/2301.00068","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2378211422","https://openalex.org/W2745001401","https://openalex.org/W4321353415","https://openalex.org/W2130974462","https://openalex.org/W2028665553","https://openalex.org/W2086519370","https://openalex.org/W972276598","https://openalex.org/W2087343574","https://openalex.org/W4246352526","https://openalex.org/W2121910908"],"abstract_inverted_index":{"Learning":[0],"to":[1,11,50,78,94,101,155],"predict":[2],"masked":[3,27,38],"tokens":[4,35],"in":[5,36,40,74,110],"a":[6,13,41,64,137],"sequence":[7],"has":[8],"been":[9],"shown":[10],"be":[12,61],"helpful":[14],"pretraining":[15],"objective":[16],"for":[17,128,148],"powerful":[18],"language":[19,28],"models":[20,29],"such":[21,26,106],"as":[22],"PaLM2.":[23],"After":[24],"training,":[25],"(MLMs)":[30],"can":[31,54,76,90],"provide":[32],"distributions":[33,48],"of":[34,112,119,132,140],"the":[37,95,146,149],"positions":[39],"sequence.":[42],"However,":[43],"this":[44],"paper":[45],"shows":[46],"that":[47,105],"corresponding":[49],"different":[51,92],"masking":[52],"patterns":[53],"demonstrate":[55],"considerable":[56,156],"inconsistencies,":[57],"i.e.,":[58],"they":[59],"cannot":[60],"derived":[62],"from":[63],"coherent":[65],"joint":[66],"distribution":[67],"when":[68],"considered":[69],"together.":[70],"This":[71],"fundamental":[72],"flaw":[73],"MLMs":[75,89,111,129],"lead":[77],"self-contradictory":[79],"behaviors":[80],"during":[81],"inference.":[82],"On":[83],"various":[84],"benchmark":[85],"datasets":[86],"including":[87],"MMLU,":[88],"give":[91],"predictions":[93],"same":[96],"input":[97],"question.":[98],"From":[99],"BERT-base":[100],"UL2-20B,":[102],"we":[103,122],"show":[104],"inconsistencies":[107],"exist":[108],"ubiquitously":[109],"diverse":[113],"sizes":[114],"and":[115],"configurations.":[116],"In":[117],"light":[118],"our":[120],"observations,":[121],"further":[123],"propose":[124],"an":[125],"inference-time":[126],"strategy":[127],"called":[130],"Ensemble":[131],"Conditionals.":[133],"It":[134],"jointly":[135],"considers":[136],"selected":[138],"range":[139],"inconsistent":[141],"conditionals":[142],"directly":[143],"produced":[144],"by":[145],"MLM":[147],"final":[150],"prediction,":[151],"which":[152],"often":[153],"leads":[154],"accuracy":[157],"improvement.":[158]},"counts_by_year":[{"year":2023,"cited_by_count":2}],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2023-01-06T00:00:00"}