{"id":"https://openalex.org/W4415202954","doi":"https://doi.org/10.48550/arxiv.2506.19780","title":"Listwise Direct Preference Optimization with Multi-Dimensional Preference Mixing","display_name":"Listwise Direct Preference Optimization with Multi-Dimensional Preference Mixing","publication_year":2025,"publication_date":"2025-06-24","ids":{"openalex":"https://openalex.org/W4415202954","doi":"https://doi.org/10.48550/arxiv.2506.19780"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2506.19780","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2506.19780","pdf_url":"https://arxiv.org/pdf/2506.19780","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2506.19780","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Sun, Yuhui","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Yuhui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114585283","display_name":"Xiyao Wang","orcid":"https://orcid.org/0009-0004-9006-0281"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Xiyao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007427445","display_name":"Zhenhong Li","orcid":"https://orcid.org/0000-0002-8054-7449"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Zixi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Ding, YiTian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ding, YiTian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Ling, Tianyang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ling, Tianyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Chen, Jialuo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Jialuo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Yu, Tianyi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu, Tianyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Yuan, Zhenlong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuan, Zhenlong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Zhao, Jinman","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Jinman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11106","display_name":"Data Management and Algorithms","score":0.9656000137329102,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11106","display_name":"Data Management and Algorithms","score":0.9656000137329102,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9616000056266785,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11063","display_name":"Rough Sets and Fuzzy Logic","score":0.9445000290870667,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.7885000109672546},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.7601000070571899},{"id":"https://openalex.org/keywords/preference-learning","display_name":"Preference learning","score":0.7249000072479248},{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.5986999869346619},{"id":"https://openalex.org/keywords/weighting","display_name":"Weighting","score":0.5569000244140625},{"id":"https://openalex.org/keywords/granularity","display_name":"Granularity","score":0.43880000710487366}],"concepts":[{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.7885000109672546},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.7601000070571899},{"id":"https://openalex.org/C181204326","wikidata":"https://www.wikidata.org/wiki/Q7239820","display_name":"Preference learning","level":3,"score":0.7249000072479248},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.5986999869346619},{"id":"https://openalex.org/C183115368","wikidata":"https://www.wikidata.org/wiki/Q856577","display_name":"Weighting","level":2,"score":0.5569000244140625},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5130000114440918},{"id":"https://openalex.org/C177774035","wikidata":"https://www.wikidata.org/wiki/Q1246948","display_name":"Granularity","level":2,"score":0.43880000710487366},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.427700012922287},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.40299999713897705},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38339999318122864},{"id":"https://openalex.org/C2779110102","wikidata":"https://www.wikidata.org/wiki/Q1323737","display_name":"Revealed preference","level":2,"score":0.35519999265670776},{"id":"https://openalex.org/C2781043087","wikidata":"https://www.wikidata.org/wiki/Q939761","display_name":"Preference theory","level":3,"score":0.35199999809265137},{"id":"https://openalex.org/C61224824","wikidata":"https://www.wikidata.org/wiki/Q2260434","display_name":"Mixture model","level":2,"score":0.323199987411499},{"id":"https://openalex.org/C194387892","wikidata":"https://www.wikidata.org/wiki/Q1747770","display_name":"Stochastic optimization","level":2,"score":0.31700000166893005},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.31060001254081726},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.30309998989105225},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.2556000053882599},{"id":"https://openalex.org/C70136482","wikidata":"https://www.wikidata.org/wiki/Q13583781","display_name":"A-weighting","level":3,"score":0.2531000077724457}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2506.19780","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2506.19780","pdf_url":"https://arxiv.org/pdf/2506.19780","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2506.19780","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2506.19780","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2506.19780","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2506.19780","pdf_url":"https://arxiv.org/pdf/2506.19780","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Recent":[0],"alignment":[1],"methods":[2,33],"based":[3,139],"on":[4,107,140,165],"Direct":[5,68],"Preference":[6,69],"Optimization":[7,70],"(DPO)":[8],"reformulate":[9],"preference":[10,39,60,81,87,99,104,121,137],"learning":[11,25],"as":[12],"supervised":[13],"optimization":[14],"over":[15,23],"pairwise":[16],"comparisons,":[17],"offering":[18],"improved":[19],"efficiency":[20],"and":[21,49,80,163],"stability":[22],"reinforcement":[24],"from":[26],"human":[27,55],"feedback":[28],"(RLHF).":[29],"However,":[30],"existing":[31],"DPO-style":[32],"implicitly":[34],"assume":[35],"a":[36,72,90,95,103,112,117,129],"single":[37,91,113],"fixed":[38],"objective,":[40],"which":[41],"limits":[42],"their":[43],"ability":[44],"to":[45,115,151],"model":[46,114,161],"the":[47,108,146,173],"structured":[48],"sometimes":[50],"conflicting":[51],"nature":[52],"of":[53,84,97,120,148],"real-world":[54],"judgments":[56],"that":[57,75,134],"span":[58],"multiple":[59,160],"dimensions.":[61],"In":[62],"this":[63],"work,":[64],"we":[65,127],"propose":[66],"Listwise":[67],"($\u03bb$-DPO),":[71],"unified":[73],"framework":[74],"simultaneously":[76],"improves":[77],"supervision":[78],"granularity":[79],"flexibility.":[82],"Instead":[83],"collapsing":[85],"multi-dimensional":[86],"signals":[88],"into":[89],"ranking,":[92],"$\u03bb$-DPO":[93],"constructs":[94],"mixture":[96],"listwise":[98],"distributions":[100],"weighted":[101],"by":[102],"vector":[105],"$\u03bb$":[106,132],"probability":[109],"simplex,":[110],"enabling":[111],"internalize":[116],"continuous":[118],"spectrum":[119],"trade-offs.":[122],"To":[123],"further":[124],"improve":[125],"robustness,":[126],"introduce":[128],"performance-driven":[130],"stochastic":[131],"scheduler":[133],"adaptively":[135],"samples":[136],"weights":[138],"empirical":[141],"downstream":[142],"performance,":[143],"explicitly":[144],"mitigating":[145],"risks":[147],"misspecification":[149],"inherent":[150],"static":[152],"weighting":[153],"schemes.":[154],"We":[155],"evaluate":[156],"our":[157],"method":[158],"across":[159],"families":[162],"scales":[164],"six":[166],"widely":[167],"used":[168],"benchmarks.":[169],"Experimental":[170],"results":[171],"show":[172],"consistent":[174],"improvement":[175],"against":[176],"baselines.":[177]},"counts_by_year":[],"updated_date":"2026-08-05T07:39:15.569665","created_date":"2025-10-16T00:00:00"}