{"id":"https://openalex.org/W4405627806","doi":"https://doi.org/10.48550/arxiv.2412.14672","title":"FiVL: A Framework for Improved Vision-Language Alignment through the Lens of Training, Evaluation and Explainability","display_name":"FiVL: A Framework for Improved Vision-Language Alignment through the Lens of Training, Evaluation and Explainability","publication_year":2024,"publication_date":"2024-12-19","ids":{"openalex":"https://openalex.org/W4405627806","doi":"https://doi.org/10.48550/arxiv.2412.14672"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2412.14672","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2412.14672","pdf_url":"https://arxiv.org/pdf/2412.14672","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2412.14672","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5011321921","display_name":"Estelle Aflalo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Aflalo, Estelle","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081879623","display_name":"Gabriela Ben Melech Stan","orcid":"https://orcid.org/0000-0001-6893-6647"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Stan, Gabriela Ben Melech","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082325147","display_name":"Tiep Le","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Le, Tiep","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101358053","display_name":"Man Luo","orcid":"https://orcid.org/0009-0008-4395-5049"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Man","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057594372","display_name":"Shachar Rosenman","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rosenman, Shachar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043910691","display_name":"Sayak Paul","orcid":"https://orcid.org/0000-0003-0217-0778"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Paul, Sayak","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003744098","display_name":"Shao-Yen Tseng","orcid":"https://orcid.org/0000-0003-1795-8330"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tseng, Shao-Yen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5087898808","display_name":"Vasudev Lal","orcid":"https://orcid.org/0000-0002-5907-9898"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lal, Vasudev","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9876999855041504,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9174000024795532,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.49260956048965454},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4319360852241516},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.35085219144821167}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.49260956048965454},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4319360852241516},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.35085219144821167}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2412.14672","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2412.14672","pdf_url":"https://arxiv.org/pdf/2412.14672","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2412.14672","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2412.14672","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2412.14672","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2412.14672","pdf_url":"https://arxiv.org/pdf/2412.14672","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4405627806.pdf","grobid_xml":"https://content.openalex.org/works/W4405627806.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2772917594","https://openalex.org/W2036807459","https://openalex.org/W2058170566","https://openalex.org/W2755342338","https://openalex.org/W2166024367","https://openalex.org/W3116076068","https://openalex.org/W2229312674","https://openalex.org/W2951359407","https://openalex.org/W2079911747","https://openalex.org/W1969923398"],"abstract_inverted_index":{"Large":[0],"Vision":[1],"Language":[2],"Models":[3],"(LVLMs)":[4],"have":[5],"achieved":[6],"significant":[7],"progress":[8],"in":[9,57,92,123,150],"integrating":[10],"visual":[11,27,55,76,95,116],"and":[12,118],"textual":[13],"inputs":[14],"for":[15,107,114],"multimodal":[16],"reasoning.":[17],"However,":[18],"a":[19,104,139],"recurring":[20],"challenge":[21],"is":[22,88,195],"ensuring":[23],"these":[24],"models":[25],"utilize":[26],"information":[28],"as":[29,31,168],"effectively":[30],"linguistic":[32,176],"content":[33],"when":[34],"both":[35],"modalities":[36],"are":[37,64],"necessary":[38],"to":[39,50,70,83,111,160,165],"formulate":[40],"an":[41],"accurate":[42],"answer.":[43],"We":[44,126],"hypothesize":[45],"that":[46,85],"hallucinations":[47],"arise":[48],"due":[49],"the":[51,68,72,75,86,128,154,162,184],"lack":[52],"of":[53,130],"effective":[54],"grounding":[56,117],"current":[58,61],"LVLMs.":[59],"Furthermore,":[60],"vision-language":[62,186],"benchmarks":[63,159],"not":[65],"specifically":[66],"measuring":[67],"degree":[69],"which":[71],"answer":[73],"require":[74],"input.":[77],"This":[78],"limitation":[79],"makes":[80],"it":[81],"challenging":[82],"confirm":[84],"image":[87,167],"truly":[89],"necessary,":[90],"particularly":[91],"tasks":[93],"like":[94],"question":[96],"answering.":[97],"In":[98],"this":[99],"work,":[100],"we":[101,137,157,179],"introduce":[102,138],"FiVL,":[103],"novel":[105,140],"method":[106],"constructing":[108],"datasets":[109,132],"designed":[110],"train":[112],"LVLMs":[113],"enhanced":[115],"also":[119],"evaluate":[120],"their":[121],"effectiveness":[122],"achieving":[124],"it.":[125],"demonstrate":[127],"value":[129],"our":[131,145],"through":[133],"three":[134],"approaches.":[135],"First,":[136],"training":[141,147],"task":[142],"based":[143],"on":[144,175,190],"augmented":[146],"dataset,":[148],"resulting":[149],"better":[151],"performance":[152],"than":[153,172],"baseline.":[155],"Second,":[156],"present":[158],"assess":[161],"model's":[163],"ability":[164],"use":[166],"substantive":[169],"evidence,":[170],"rather":[171],"relying":[173],"solely":[174],"priors.":[177],"Finally,":[178],"identify":[180],"attention":[181],"heads":[182],"with":[183],"strongest":[185],"alignment,":[187],"enabling":[188],"explainability":[189],"visual-driven":[191],"hallucinations.":[192],"The":[193],"code":[194],"available":[196],"at":[197],"https://github.com/IntelLabs/fivl.":[198]},"counts_by_year":[],"updated_date":"2026-08-21T09:56:20.448147","created_date":"2024-12-21T00:00:00"}