{"id":"https://openalex.org/W4405622408","doi":"https://doi.org/10.48550/arxiv.2408.12249","title":"LLMs are not Zero-Shot Reasoners for Biomedical Information Extraction","display_name":"LLMs are not Zero-Shot Reasoners for Biomedical Information Extraction","publication_year":2024,"publication_date":"2024-08-22","ids":{"openalex":"https://openalex.org/W4405622408","doi":"https://doi.org/10.48550/arxiv.2408.12249"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2408.12249","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2408.12249","pdf_url":"https://arxiv.org/pdf/2408.12249","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2408.12249","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113407769","display_name":"Aishik Nagar","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Nagar, Aishik","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037328707","display_name":"Viktor Schlegel","orcid":"https://orcid.org/0000-0002-6391-2950"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Schlegel, Viktor","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101924638","display_name":"Thanh-Tung Nguyen","orcid":"https://orcid.org/0000-0002-8186-2600"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nguyen, Thanh-Tung","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100348631","display_name":"Hao Li","orcid":"https://orcid.org/0000-0002-7577-1366"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Hao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101470492","display_name":"Yuping Wu","orcid":"https://orcid.org/0000-0001-6399-9710"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Yuping","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021254326","display_name":"Kuluhan Binici","orcid":"https://orcid.org/0000-0003-0295-0547"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Binici, Kuluhan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5018017185","display_name":"Stefan Winkler","orcid":"https://orcid.org/0000-0003-4305-8408"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Winkler, Stefan","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5113407769"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.954200029373169,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.954200029373169,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/shot","display_name":"Shot (pellet)","score":0.7252029776573181},{"id":"https://openalex.org/keywords/zero","display_name":"Zero (linguistics)","score":0.684887170791626},{"id":"https://openalex.org/keywords/extraction","display_name":"Extraction (chemistry)","score":0.5809566974639893},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.32208770513534546},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.25743699073791504},{"id":"https://openalex.org/keywords/materials-science","display_name":"Materials science","score":0.2041986584663391},{"id":"https://openalex.org/keywords/chromatography","display_name":"Chromatography","score":0.17591917514801025},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.11899960041046143},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.09549224376678467},{"id":"https://openalex.org/keywords/metallurgy","display_name":"Metallurgy","score":0.07664105296134949}],"concepts":[{"id":"https://openalex.org/C2778344882","wikidata":"https://www.wikidata.org/wiki/Q278938","display_name":"Shot (pellet)","level":2,"score":0.7252029776573181},{"id":"https://openalex.org/C2780813799","wikidata":"https://www.wikidata.org/wiki/Q3274237","display_name":"Zero (linguistics)","level":2,"score":0.684887170791626},{"id":"https://openalex.org/C4725764","wikidata":"https://www.wikidata.org/wiki/Q844704","display_name":"Extraction (chemistry)","level":2,"score":0.5809566974639893},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.32208770513534546},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.25743699073791504},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.2041986584663391},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.17591917514801025},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.11899960041046143},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.09549224376678467},{"id":"https://openalex.org/C191897082","wikidata":"https://www.wikidata.org/wiki/Q11467","display_name":"Metallurgy","level":1,"score":0.07664105296134949}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2408.12249","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2408.12249","pdf_url":"https://arxiv.org/pdf/2408.12249","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2408.12249","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2408.12249","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2408.12249","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2408.12249","pdf_url":"https://arxiv.org/pdf/2408.12249","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4405622408.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2074502265","https://openalex.org/W4214877189","https://openalex.org/W2773965352","https://openalex.org/W2381179799","https://openalex.org/W2980279061","https://openalex.org/W2334685461","https://openalex.org/W2366718574"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"are":[4,42,201,212],"increasingly":[5],"adopted":[6],"for":[7,191,218],"applications":[8],"in":[9,45,58,66,170,179,228,234],"healthcare,":[10],"reaching":[11],"the":[12,46,79,85,88,168,171,180,216],"performance":[13,65,233],"of":[14,81,90,103,125,174,222],"domain":[15,99],"experts":[16],"on":[17,29,39,121],"tasks":[18,40,207],"such":[19,49,196],"as":[20,50,139,141,197],"question":[21],"answering":[22],"and":[23,69,94,101,117,134,147,177,225],"document":[24],"summarisation.":[25],"Despite":[26],"their":[27,97,232],"success":[28],"these":[30],"tasks,":[31,165,195],"it":[32],"is":[33],"unclear":[34],"how":[35],"well":[36,140],"LLMs":[37,113,229],"perform":[38],"that":[41,155,186],"traditionally":[43],"pursued":[44],"biomedical":[47,126,181,206,236],"domain,":[48],"structured":[51,210],"information":[52],"extraction.":[53],"To":[54,106],"bridge":[55],"this":[56,59,107],"gap,":[57],"paper,":[60],"we":[61,109],"systematically":[62],"benchmark":[63],"LLM":[64],"Medical":[67],"Classification":[68],"Named":[70],"Entity":[71],"Recognition":[72],"(NER)":[73],"tasks.":[74],"We":[75],"aim":[76],"to":[77,84,205,230],"disentangle":[78],"contribution":[80],"different":[82],"factors":[83],"performance,":[86],"particularly":[87],"impact":[89],"LLMs'":[91],"task":[92],"knowledge":[93,224],"reasoning":[95,138,226],"capabilities,":[96],"(parametric)":[98],"knowledge,":[100],"addition":[102],"external":[104,223],"knowledge.":[105],"end,":[108],"evaluate":[110],"various":[111],"open":[112],"-":[114,120],"including":[115],"BioMistral":[116],"Llama-2":[118],"models":[119],"a":[122],"diverse":[123],"set":[124],"datasets,":[127],"using":[128],"standard":[129,156],"prompting,":[130],"Chain":[131],"of-Thought":[132],"(CoT)":[133],"Self":[135],"Consistency":[136],"based":[137],"Retrieval-Augmented":[142],"Generation":[143],"(RAG)":[144],"with":[145],"PubMed":[146],"Wikipedia":[148],"corpora.":[149],"Counter":[150],"intuitively,":[151],"our":[152],"results":[153],"reveal":[154],"prompting":[157,188],"consistently":[158],"outperforms":[159],"more":[160,219],"complex":[161],"techniques":[162],"across":[163],"both":[164],"laying":[166],"bare":[167],"limitations":[169],"current":[172],"application":[173],"CoT,":[175],"self-consistency":[176],"RAG":[178],"domain.":[182],"Our":[183],"findings":[184],"suggest":[185],"advanced":[187],"methods":[189],"developed":[190],"knowledge-":[192],"or":[193,199],"reasoning-intensive":[194],"CoT":[198],"RAG,":[200],"not":[202],"easily":[203],"portable":[204],"where":[208],"precise":[209],"outputs":[211],"required.":[213],"This":[214],"highlights":[215],"need":[217],"effective":[220],"integration":[221],"mechanisms":[227],"enhance":[231],"real-world":[235],"applications.":[237]},"counts_by_year":[],"updated_date":"2026-03-11T14:59:36.786465","created_date":"2025-10-10T00:00:00"}