{"id":"https://openalex.org/W7129099413","doi":"https://doi.org/10.48550/arxiv.2602.12966","title":"ProbeLLM: Automating Principled Diagnosis of LLM Failures","display_name":"ProbeLLM: Automating Principled Diagnosis of LLM Failures","publication_year":2026,"publication_date":"2026-02-13","ids":{"openalex":"https://openalex.org/W7129099413","doi":"https://doi.org/10.48550/arxiv.2602.12966"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2602.12966","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.12966","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2602.12966","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5126113312","display_name":"Yue Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Yue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126123208","display_name":"Zhengzhe Jiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiang, Zhengzhe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126146966","display_name":"Yuchen Ma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Yuchen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126169829","display_name":"Yu Jiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiang, Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5146663499","display_name":"Xiangqi Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Xiangqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126116369","display_name":"Yujun Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Yujun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004826137","display_name":"Yuexing Hao","orcid":"https://orcid.org/0000-0003-4375-7655"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hao, Yuexing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5122939434","display_name":"Kehan Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Kehan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126142840","display_name":"Pin-Yu Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Pin-Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126135385","display_name":"Stefan Feuerriegel","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Feuerriegel, Stefan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5126163040","display_name":"Xiangliang Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Xiangliang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.25429999828338623,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.25429999828338623,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.2087000012397766,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.05979999899864197,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/verifiable-secret-sharing","display_name":"Verifiable secret sharing","score":0.4586000144481659},{"id":"https://openalex.org/keywords/tree","display_name":"Tree (set theory)","score":0.3797999918460846},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.34150001406669617},{"id":"https://openalex.org/keywords/failure-mode-and-effects-analysis","display_name":"Failure mode and effects analysis","score":0.3095000088214874},{"id":"https://openalex.org/keywords/automation","display_name":"Automation","score":0.2971000075340271},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.2946999967098236}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7075999975204468},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.4586000144481659},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.41339999437332153},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4081999957561493},{"id":"https://openalex.org/C113174947","wikidata":"https://www.wikidata.org/wiki/Q2859736","display_name":"Tree (set theory)","level":2,"score":0.3797999918460846},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.34150001406669617},{"id":"https://openalex.org/C66283442","wikidata":"https://www.wikidata.org/wiki/Q1389268","display_name":"Failure mode and effects analysis","level":2,"score":0.3095000088214874},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.2971000075340271},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.2946999967098236},{"id":"https://openalex.org/C110251889","wikidata":"https://www.wikidata.org/wiki/Q1569697","display_name":"Model checking","level":2,"score":0.2912999987602234},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.2542000114917755},{"id":"https://openalex.org/C19499675","wikidata":"https://www.wikidata.org/wiki/Q232207","display_name":"Monte Carlo method","level":2,"score":0.2535000145435333},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.2513999938964844}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2602.12966","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.12966","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2602.12966","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.12966","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Understanding":[0],"how":[1],"and":[2,18,45,97,111,115,136,142,149,157],"why":[3],"large":[4],"language":[5],"models":[6,15],"(LLMs)":[7],"fail":[8],"is":[9],"becoming":[10],"a":[11,59,79,162],"central":[12],"challenge":[13],"as":[14,78],"rapidly":[16],"evolve":[17],"static":[19,155],"evaluations":[20],"fall":[21],"behind.":[22],"While":[23],"automated":[24,61,159],"probing":[25,62,77,88,106],"has":[26],"been":[27],"enabled":[28],"by":[29],"dynamic":[30],"test":[31,109],"generation,":[32],"existing":[33],"approaches":[34],"often":[35],"discover":[36],"isolated":[37],"failure":[38,73,95,119,131,152],"cases,":[39],"lack":[40],"principled":[41,168],"control":[42],"over":[43],"exploration,":[44],"provide":[46],"limited":[47,87],"insight":[48],"into":[49,129],"the":[50],"underlying":[51],"structure":[52],"of":[53,93,100],"model":[54],"weaknesses.":[55],"We":[56],"propose":[57],"ProbeLLM,":[58],"benchmark-agnostic":[60],"framework":[63],"that":[64],"elevates":[65],"weakness":[66,169],"discovery":[67,120],"from":[68,164],"individual":[69],"failures":[70,125],"to":[71,107],"structured":[72],"modes.":[74],"ProbeLLM":[75,117,144],"formulates":[76],"hierarchical":[80],"Monte":[81],"Carlo":[82],"Tree":[83],"Search,":[84],"explicitly":[85],"allocating":[86],"budgets":[89],"between":[90],"global":[91],"exploration":[92],"new":[94],"regions":[96],"local":[98],"refinement":[99],"recurring":[101],"error":[102],"patterns.":[103],"By":[104],"restricting":[105],"verifiable":[108],"cases":[110],"leveraging":[112],"tool-augmented":[113],"generation":[114],"verification,":[116],"grounds":[118],"in":[121],"reliable":[122],"evidence.":[123],"Discovered":[124],"are":[126],"further":[127],"consolidated":[128],"interpretable":[130],"modes":[132],"via":[133],"failure-aware":[134],"embeddings":[135],"boundary-aware":[137],"induction.":[138],"Across":[139],"diverse":[140],"benchmarks":[141,156],"LLMs,":[143],"reveals":[145],"substantially":[146],"broader,":[147],"cleaner,":[148],"more":[150],"fine-grained":[151],"landscapes":[153],"than":[154],"prior":[158],"methods,":[160],"supporting":[161],"shift":[163],"case-centric":[165],"evaluation":[166],"toward":[167],"discovery.":[170]},"counts_by_year":[],"updated_date":"2026-08-16T07:02:28.622633","created_date":"2026-02-17T00:00:00"}