{"id":"https://openalex.org/W4403574147","doi":"https://doi.org/10.48550/arxiv.2410.11462","title":"Mitigating Frequency Bias and Anisotropy in Language Model Pre-Training with Syntactic Smoothing","display_name":"Mitigating Frequency Bias and Anisotropy in Language Model Pre-Training with Syntactic Smoothing","publication_year":2024,"publication_date":"2024-10-15","ids":{"openalex":"https://openalex.org/W4403574147","doi":"https://doi.org/10.48550/arxiv.2410.11462"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2410.11462","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.11462","pdf_url":"https://arxiv.org/pdf/2410.11462","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2410.11462","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5027311833","display_name":"Richard Diehl Martinez","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Martinez, Richard Diehl","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092850350","display_name":"Z\u00e9bulon Goriely","orcid":"https://orcid.org/0000-0002-8877-4099"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Goriely, Zebulon","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069029343","display_name":"Andrew Caines","orcid":"https://orcid.org/0000-0001-9647-4902"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Caines, Andrew","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013999263","display_name":"Paula Buttery","orcid":"https://orcid.org/0000-0003-3874-0656"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Buttery, Paula","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5087195265","display_name":"Lisa Beinborn","orcid":"https://orcid.org/0000-0001-8654-2190"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Beinborn, Lisa","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9649999737739563,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9649999737739563,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9643999934196472,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9000999927520752,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/smoothing","display_name":"Smoothing","score":0.7788487672805786},{"id":"https://openalex.org/keywords/anisotropy","display_name":"Anisotropy","score":0.5931007266044617},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5125601291656494},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.4298958480358124},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4078770875930786},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.37713152170181274},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.1811905801296234},{"id":"https://openalex.org/keywords/optics","display_name":"Optics","score":0.1338508129119873},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.06585273146629333},{"id":"https://openalex.org/keywords/meteorology","display_name":"Meteorology","score":0.05667153000831604}],"concepts":[{"id":"https://openalex.org/C3770464","wikidata":"https://www.wikidata.org/wiki/Q775963","display_name":"Smoothing","level":2,"score":0.7788487672805786},{"id":"https://openalex.org/C85725439","wikidata":"https://www.wikidata.org/wiki/Q466686","display_name":"Anisotropy","level":2,"score":0.5931007266044617},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5125601291656494},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.4298958480358124},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4078770875930786},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37713152170181274},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.1811905801296234},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.1338508129119873},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.06585273146629333},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.05667153000831604}],"mesh":[],"locations_count":4,"locations":[{"id":"pmh:oai:arXiv.org:2410.11462","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.11462","pdf_url":"https://arxiv.org/pdf/2410.11462","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:publications.goettingen-research-online.de:2/152720","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.11462v1","pdf_url":"https://arxiv.org/pdf/2410.11462","source":{"id":"https://openalex.org/S4306401634","display_name":"GoeScholar  The Publication Server of the Georg-August-Universit\u00e4t G\u00f6ttingen (Georg-August-Universit\u00e4t G\u00f6ttingen)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210122495","host_organization_name":"Asklepios Klinik St. Georg","host_organization_lineage":["https://openalex.org/I4210122495"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:publications.goettingen-research-online.de:2/152703","is_oa":true,"landing_page_url":"https://resolver.sub.uni-goettingen.de/purl?gro-2/152703","pdf_url":null,"source":{"id":"https://openalex.org/S4306401634","display_name":"GoeScholar  The Publication Server of the Georg-August-Universit\u00e4t G\u00f6ttingen (Georg-August-Universit\u00e4t G\u00f6ttingen)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210122495","host_organization_name":"Asklepios Klinik St. Georg","host_organization_lineage":["https://openalex.org/I4210122495"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/preprint"},{"id":"doi:10.48550/arxiv.2410.11462","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2410.11462","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2410.11462","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.11462","pdf_url":"https://arxiv.org/pdf/2410.11462","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1690597291","display_name":null,"funder_award_id":"#OPP1144","funder_id":"https://openalex.org/F4320323264","funder_display_name":"Gates Cambridge Trust"},{"id":"https://openalex.org/G5477836236","display_name":"Cambridge University Development Office in the United States Inc (CUDOUS) - to endow a global scholarship program at the post-baccalaureate level for academically gifted students","funder_award_id":"OPP1144","funder_id":"https://openalex.org/F4320306137","funder_display_name":"Bill and Melinda Gates Foundation"}],"funders":[{"id":"https://openalex.org/F4320306137","display_name":"Bill and Melinda Gates Foundation","ror":"https://ror.org/0456r8d26"},{"id":"https://openalex.org/F4320312927","display_name":"Dell EMC","ror":"https://ror.org/05rejmm18"},{"id":"https://openalex.org/F4320320273","display_name":"University of Cambridge","ror":"https://ror.org/013meh722"},{"id":"https://openalex.org/F4320321800","display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","ror":"https://ror.org/04jsz6e67"},{"id":"https://openalex.org/F4320321848","display_name":"Cambridge Trust","ror":"https://ror.org/05sprqy15"},{"id":"https://openalex.org/F4320323264","display_name":"Gates Cambridge Trust","ror":"https://ror.org/033sn5p83"},{"id":"https://openalex.org/F4320334632","display_name":"Science and Technology Facilities Council","ror":"https://ror.org/057g20z61"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403574147.pdf","grobid_xml":"https://content.openalex.org/works/W4403574147.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W1978572805","https://openalex.org/W2383807498","https://openalex.org/W1997992934","https://openalex.org/W1987225439","https://openalex.org/W4238188170","https://openalex.org/W2125114371","https://openalex.org/W2019977573","https://openalex.org/W2149980199","https://openalex.org/W230091440","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Language":[0],"models":[1,20],"strongly":[2],"rely":[3],"on":[4,140],"frequency":[5,76,99,163],"information":[6],"because":[7],"they":[8],"maximize":[9],"the":[10,75,98,120,127,153],"likelihood":[11,36,122],"of":[12,47,78,101,155],"tokens":[13,27,48,143],"during":[14,32,113],"pre-training.":[15,114],"As":[16],"a":[17,50,57,71,79,94,102,107,145,158],"consequence,":[18],"language":[19,80,103],"tend":[21,52],"to":[22,26,41,44,53,88,125,130],"not":[23],"generalize":[24],"well":[25],"that":[28,152],"are":[29],"seldom":[30],"seen":[31],"training.":[33],"Moreover,":[34],"maximum":[35,121],"training":[37],"has":[38],"been":[39],"discovered":[40],"give":[42],"rise":[43],"anisotropy:":[45],"representations":[46,112],"in":[49,56,137,147,157],"model":[51,81,104,159],"cluster":[54],"tightly":[55],"high-dimensional":[58],"cone,":[59],"rather":[60],"than":[61],"spreading":[62],"out":[63],"over":[64,110],"their":[65],"representational":[66],"capacity.":[67],"Our":[68,115],"work":[69],"introduces":[70],"method":[72,95,118],"for":[73,96],"quantifying":[74],"bias":[77,100],"by":[82,105],"assessing":[83],"sentence-level":[84],"perplexity":[85],"with":[86,161],"respect":[87],"token-level":[89],"frequency.":[90],"We":[91,149],"then":[92],"present":[93],"reducing":[97],"inducing":[106],"syntactic":[108],"prior":[109],"token":[111],"Syntactic":[116],"Smoothing":[117],"adjusts":[119],"objective":[123],"function":[124],"distribute":[126],"learning":[128],"signal":[129],"syntactically":[131],"similar":[132],"tokens.":[133],"This":[134],"approach":[135],"results":[136],"better":[138],"performance":[139],"infrequent":[141],"English":[142],"and":[144],"decrease":[146],"anisotropy.":[148],"empirically":[150],"show":[151],"degree":[154],"anisotropy":[156],"correlates":[160],"its":[162],"bias.":[164]},"counts_by_year":[],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2025-10-10T00:00:00"}