{"id":"https://openalex.org/W4399911164","doi":"https://doi.org/10.48550/arxiv.2406.13743","title":"GenAI-Bench: Evaluating and Improving Compositional Text-to-Visual Generation","display_name":"GenAI-Bench: Evaluating and Improving Compositional Text-to-Visual Generation","publication_year":2024,"publication_date":"2024-06-19","ids":{"openalex":"https://openalex.org/W4399911164","doi":"https://doi.org/10.48550/arxiv.2406.13743"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2406.13743","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.13743","pdf_url":"https://arxiv.org/pdf/2406.13743","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2406.13743","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101541888","display_name":"Baiqi Li","orcid":"https://orcid.org/0000-0003-0764-3159"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Baiqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100575316","display_name":"Zhiqiu Lin","orcid":"https://orcid.org/0000-0003-4938-2313"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Zhiqiu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101851026","display_name":"Deepak Pathak","orcid":"https://orcid.org/0000-0003-2496-0690"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pathak, Deepak","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100603319","display_name":"Jiayao Li","orcid":"https://orcid.org/0009-0002-9505-7343"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Jiayao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023679731","display_name":"Yixin Fei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fei, Yixin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087597197","display_name":"Kewen Wu","orcid":"https://orcid.org/0000-0002-5894-822X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Kewen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5099380649","display_name":"Tiffany Ling","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ling, Tiffany","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091179204","display_name":"Xide Xia","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xia, Xide","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059735251","display_name":"Pengchuan Zhang","orcid":"https://orcid.org/0000-0003-1155-9507"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Pengchuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068811427","display_name":"Graham Neubig","orcid":"https://orcid.org/0000-0002-2072-3789"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Neubig, Graham","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5004353237","display_name":"Deva Ramanan","orcid":"https://orcid.org/0009-0008-9180-8983"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ramanan, Deva","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9578999876976013,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9578999876976013,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9391999840736389,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13071","display_name":"Digital Storytelling and Education","score":0.9100000262260437,"subfield":{"id":"https://openalex.org/subfields/3616","display_name":"Speech and Hearing"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4976096451282501},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4727160334587097}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4976096451282501},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4727160334587097}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2406.13743","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.13743","pdf_url":"https://arxiv.org/pdf/2406.13743","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2406.13743","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2406.13743","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2406.13743","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.13743","pdf_url":"https://arxiv.org/pdf/2406.13743","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2382290278","https://openalex.org/W4395014643"],"abstract_inverted_index":{"While":[0],"text-to-visual":[1,53],"models":[2,47,209],"now":[3],"produce":[4],"photo-realistic":[5],"images":[6,173],"and":[7,18,24,44,66,134,144,210],"videos,":[8],"they":[9],"struggle":[10],"with":[11,162],"compositional":[12,52,149],"text":[13],"prompts":[14,150],"involving":[15],"attributes,":[16],"relationships,":[17],"higher-order":[19],"reasoning":[20],"such":[21,93,188],"as":[22,83,94,189],"logic":[23],"comparison.":[25],"In":[26,96],"this":[27],"work,":[28],"we":[29,180],"conduct":[30],"an":[31,81],"extensive":[32],"human":[33,64,138,165,198],"study":[34],"on":[35,148,171],"GenAI-Bench":[36],"to":[37,114,123,167,202],"evaluate":[38,168],"the":[39,74,86,176],"performance":[40],"of":[41,51,206],"leading":[42],"image":[43,82],"video":[45],"generation":[46,101],"in":[48,102,186],"various":[49],"aspects":[50],"generation.":[54],"We":[55,156,194],"also":[56],"compare":[57],"automated":[58,211],"evaluation":[59],"metrics":[60,92,170],"against":[61],"our":[62],"collected":[63],"ratings":[65,140,166,199],"find":[67],"that":[68,76,151],"VQAScore":[69,98,120],"--":[70,88],"a":[71,77,103,111,158],"metric":[72],"measuring":[73],"likelihood":[75],"VQA":[78],"model":[79],"views":[80],"accurately":[84],"depicting":[85],"prompt":[87],"significantly":[89],"outperforms":[90],"previous":[91],"CLIPScore.":[95],"addition,":[97],"can":[99],"improve":[100],"black-box":[104],"manner":[105],"(without":[106],"finetuning)":[107],"via":[108],"simply":[109],"ranking":[110,172],"few":[112],"(3":[113],"9)":[115],"candidate":[116],"images.":[117],"Ranking":[118],"by":[119],"is":[121],"2x":[122],"3x":[124],"more":[125],"effective":[126],"than":[127],"other":[128],"scoring":[129,169],"methods":[130],"like":[131],"PickScore,":[132],"HPSv2,":[133],"ImageReward":[135],"at":[136],"improving":[137],"alignment":[139],"for":[141,184],"DALL-E":[142],"3":[143],"Stable":[145],"Diffusion,":[146],"especially":[147],"require":[152],"advanced":[153],"visio-linguistic":[154],"reasoning.":[155],"release":[157,196],"new":[159],"GenAI-Rank":[160],"benchmark":[161],"over":[163],"40,000":[164],"generated":[174],"from":[175],"same":[177],"prompt.":[178],"Lastly,":[179],"discuss":[181],"promising":[182],"areas":[183],"improvement":[185],"VQAScore,":[187],"addressing":[190],"fine-grained":[191],"visual":[192],"details.":[193],"will":[195],"all":[197],"(over":[200],"80,000)":[201],"facilitate":[203],"scientific":[204],"benchmarking":[205],"both":[207],"generative":[208],"metrics.":[212]},"counts_by_year":[],"updated_date":"2026-08-11T07:18:39.950985","created_date":"2024-06-22T00:00:00"}