{"id":"https://openalex.org/W4304699873","doi":"https://doi.org/10.48550/arxiv.2210.03871","title":"Data-Efficiency with a Single GPU: An Exploration of Transfer Methods for Small Language Models","display_name":"Data-Efficiency with a Single GPU: An Exploration of Transfer Methods for Small Language Models","publication_year":2022,"publication_date":"2022-10-08","ids":{"openalex":"https://openalex.org/W4304699873","doi":"https://doi.org/10.48550/arxiv.2210.03871"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2210.03871","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2210.03871","pdf_url":"https://arxiv.org/pdf/2210.03871","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2210.03871","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5090026363","display_name":"Alon Albalak","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Albalak, Alon","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045502911","display_name":"Akshat Shrivastava","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shrivastava, Akshat","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039984997","display_name":"Chinnadhurai Sankar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sankar, Chinnadhurai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113432231","display_name":"Adithya Sagar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sagar, Adithya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5112196138","display_name":"Mike Ross","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ross, Mike","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9909999966621399,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/generalizability-theory","display_name":"Generalizability theory","score":0.883724570274353},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7712243795394897},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.6741336584091187},{"id":"https://openalex.org/keywords/transfer-of-learning","display_name":"Transfer of learning","score":0.5176451802253723},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5144398808479309},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5123695135116577},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4763900637626648},{"id":"https://openalex.org/keywords/fine-tuning","display_name":"Fine-tuning","score":0.4556165337562561},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.43406301736831665},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.39767181873321533},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.14774489402770996},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09274753928184509}],"concepts":[{"id":"https://openalex.org/C27158222","wikidata":"https://www.wikidata.org/wiki/Q5532422","display_name":"Generalizability theory","level":2,"score":0.883724570274353},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7712243795394897},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.6741336584091187},{"id":"https://openalex.org/C150899416","wikidata":"https://www.wikidata.org/wiki/Q1820378","display_name":"Transfer of learning","level":2,"score":0.5176451802253723},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5144398808479309},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5123695135116577},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4763900637626648},{"id":"https://openalex.org/C157524613","wikidata":"https://www.wikidata.org/wiki/Q2828883","display_name":"Fine-tuning","level":2,"score":0.4556165337562561},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.43406301736831665},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.39767181873321533},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.14774489402770996},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09274753928184509},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2210.03871","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2210.03871","pdf_url":"https://arxiv.org/pdf/2210.03871","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2210.03871","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2210.03871","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2210.03871","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2210.03871","pdf_url":"https://arxiv.org/pdf/2210.03871","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.8199999928474426,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W3023285645","https://openalex.org/W3037551068","https://openalex.org/W3023594376","https://openalex.org/W4287802662","https://openalex.org/W3126778358","https://openalex.org/W3154646238","https://openalex.org/W4378942230","https://openalex.org/W2735783528","https://openalex.org/W4220759178","https://openalex.org/W4309877123"],"abstract_inverted_index":{"Multi-task":[0],"learning":[1],"(MTL),":[2],"instruction":[3,62,114],"tuning,":[4,63],"and":[5,46,64],"prompting":[6],"have":[7],"recently":[8],"been":[9],"shown":[10],"to":[11,19,105],"improve":[12],"the":[13,23,48,79],"generalizability":[14],"of":[15,25,50],"large":[16,109],"language":[17,33],"models":[18,69,84],"new":[20],"tasks.":[21],"However,":[22],"benefits":[24],"such":[26],"methods":[27],"are":[28],"less":[29],"well-documented":[30],"in":[31,78],"smaller":[32],"models,":[34,110],"with":[35,70,95],"some":[36],"studies":[37],"finding":[38],"contradictory":[39],"results.":[40],"In":[41],"this":[42],"work,":[43],"we":[44,111],"explore":[45],"isolate":[47],"effects":[49],"(i)":[51],"model":[52],"size,":[53],"(ii)":[54],"general":[55,92],"purpose":[56,93],"MTL,":[57,60,94],"(iii)":[58],"in-domain":[59,102],"(iv)":[61],"(v)":[65],"few-shot":[66],"fine-tuning":[67],"for":[68,122],"fewer":[71],"than":[72],"500":[73],"million":[74],"parameters.":[75],"Our":[76],"experiments":[77],"zero-shot":[80],"setting":[81],"demonstrate":[82],"that":[83,113],"gain":[85,100],"31%":[86],"relative":[87,99],"improvement,":[88],"on":[89,108],"average,":[90],"from":[91,101],"an":[96],"additional":[97],"37.6%":[98],"MTL.":[103],"Contradictory":[104],"prior":[106],"works":[107],"find":[112],"tuning":[115],"provides":[116],"a":[117],"modest":[118],"2%":[119],"performance":[120],"improvement":[121],"small":[123],"models.":[124]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2026-08-11T07:18:39.950985","created_date":"2022-10-12T00:00:00"}