{"id":"https://openalex.org/W4416059534","doi":"https://doi.org/10.63317/46t2asmno5ng","title":"Redefining Evaluation Standards: A Unified Framework for Evaluating the Korean Capabilities of Language Models","display_name":"Redefining Evaluation Standards: A Unified Framework for Evaluating the Korean Capabilities of Language Models","publication_year":2026,"publication_date":"2026-04-30","ids":{"openalex":"https://openalex.org/W4416059534","doi":"https://doi.org/10.63317/46t2asmno5ng"},"language":"en","primary_location":{"id":"doi:10.63317/46t2asmno5ng","is_oa":false,"landing_page_url":"https://doi.org/10.63317/46t2asmno5ng","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Language Resources and Evaluation Conference","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2503.22968","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103110871","display_name":"Hanwool Lee","orcid":"https://orcid.org/0000-0001-5565-5184"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hanwool Lee","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113082664","display_name":"Dasol Choi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dasol Choi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102755328","display_name":"Sooyong Kim","orcid":"https://orcid.org/0000-0002-5131-5892"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sooyong Kim","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Ilgyun Jung","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ilgyun Jung","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046244115","display_name":"Sangwon Baek","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sangwon Baek","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028962243","display_name":"Guijin Son","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guijin Son","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120321521","display_name":"Inseon Hwang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Inseong Hwang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103047016","display_name":"Naeun Lee","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Naeun Lee","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5110443261","display_name":"Seunghyeok Hong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Seunghyeok Hong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.02191235,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"4414","last_page":"4422"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.34769999980926514,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.34769999980926514,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.16500000655651093,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.09489999711513519,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.7641000151634216},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.6421999931335449},{"id":"https://openalex.org/keywords/modular-design","display_name":"Modular design","score":0.5447999835014343},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5270000100135803},{"id":"https://openalex.org/keywords/semantics","display_name":"Semantics (computer science)","score":0.3573000133037567},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.3508000075817108}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7714999914169312},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.7641000151634216},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.6421999931335449},{"id":"https://openalex.org/C101468663","wikidata":"https://www.wikidata.org/wiki/Q1620158","display_name":"Modular design","level":2,"score":0.5447999835014343},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5270000100135803},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4641000032424927},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3959999978542328},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.3573000133037567},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3508000075817108},{"id":"https://openalex.org/C188087704","wikidata":"https://www.wikidata.org/wiki/Q369577","display_name":"Standardization","level":2,"score":0.34450000524520874},{"id":"https://openalex.org/C2779777834","wikidata":"https://www.wikidata.org/wiki/Q4202277","display_name":"Enforcement","level":2,"score":0.34150001406669617},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.3391000032424927},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.33070001006126404},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3165000081062317},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2847999930381775},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.2728999853134155},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.27129998803138733},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.26409998536109924}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.63317/46t2asmno5ng","is_oa":false,"landing_page_url":"https://doi.org/10.63317/46t2asmno5ng","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Language Resources and Evaluation Conference","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2503.22968","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2503.22968","pdf_url":"https://arxiv.org/pdf/2503.22968","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2503.22968","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2503.22968","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2503.22968","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2503.22968","pdf_url":"https://arxiv.org/pdf/2503.22968","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Recent":[0],"advancements":[1],"in":[2,153,159],"Korean":[3,68,74,89,160],"large":[4],"language":[5,83],"models":[6],"(LLMs)":[7],"have":[8],"driven":[9],"numerous":[10],"benchmarks":[11],"and":[12,45,79,103,129,150],"evaluation":[13],"methods,":[14,102],"yet":[15],"inconsistent":[16],"protocols":[17],"cause":[18],"up":[19],"to":[20,50,86,109],"10":[21],"p.p":[22],"performance":[23],"gaps":[24,30],"across":[25],"institutions.":[26],"Overcoming":[27],"these":[28],"reproducibility":[29],"does":[31],"not":[32],"mean":[33],"enforcing":[34],"a":[35,46],"one-size-fits-all":[36],"evaluation.":[37],"Rather,":[38],"effective":[39],"benchmarking":[40],"requires":[41],"diverse":[42],"experimental":[43],"approaches":[44],"framework":[47,65],"robust":[48],"enough":[49],"support":[51],"them.":[52],"To":[53],"this":[54],"end,":[55],"we":[56],"introduce":[57],"HRET":[58,71,117],"(Haerae":[59],"Evaluation":[60],"Toolkit),":[61],"an":[62],"open-source,":[63],"registry-based":[64],"that":[66],"unifies":[67],"LLM":[69,161],"assessment.":[70],"integrates":[72],"major":[73],"benchmarks,":[75],"multiple":[76],"inference":[77],"backends,":[78,104],"multi-method":[80],"evaluation,":[81],"with":[82],"consistency":[84],"enforcement":[85],"ensure":[87],"genuine":[88],"outputs.":[90],"Its":[91],"modular":[92],"registry":[93],"design":[94],"also":[95],"enables":[96],"rapid":[97],"incorporation":[98],"of":[99],"new":[100],"datasets,":[101],"ensuring":[105],"the":[106],"toolkit":[107],"adapts":[108],"evolving":[110],"research":[111],"needs.":[112],"Beyond":[113],"standard":[114],"accuracy":[115],"metrics,":[116],"incorporates":[118],"Korean-focused":[119],"output":[120],"analyses-morphology-aware":[121],"Type-Token":[122],"Ratio":[123],"(TTR)":[124],"for":[125,133],"evaluating":[126],"lexical":[127],"diversity":[128],"systematic":[130],"keyword-omission":[131],"detection":[132],"identifying":[134],"missing":[135],"concepts-to":[136],"provide":[137],"diagnostic":[138],"insights":[139],"into":[140],"language-specific":[141],"behaviors.":[142],"These":[143],"targeted":[144],"analyses":[145],"help":[146],"researchers":[147],"pinpoint":[148],"morphological":[149],"semantic":[151],"shortcomings":[152],"model":[154],"outputs,":[155],"guiding":[156],"focused":[157],"improvements":[158],"development.":[162]},"counts_by_year":[],"updated_date":"2026-08-16T07:02:28.622633","created_date":"2025-10-10T00:00:00"}