@inproceedings{pruksachatkun-etal-2020-intermediate,
    title = "Intermediate-Task Transfer Learning with Pretrained Language Models: When and Why Does It Work?",
    author = "Pruksachatkun, Yada  and
      Phang, Jason  and
      Liu, Haokun  and
      Htut, Phu Mon  and
      Zhang, Xiaoyi  and
      Pang, Richard Yuanzhe  and
      Vania, Clara  and
      Kann, Katharina  and
      Bowman, Samuel R.",
    booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics",
    month = jul,
    year = "2020",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://www.aclweb.org/anthology/2020.acl-main.467",
    pages = "5231--5247",
    abstract = "While pretrained models such as BERT have shown large gains across natural language understanding tasks, their performance can be improved by further training the model on a data-rich intermediate task, before fine-tuning it on a target task. However, it is still poorly understood when and why intermediate-task training is beneficial for a given target task. To investigate this, we perform a large-scale study on the pretrained RoBERTa model with 110 intermediate-target task combinations. We further evaluate all trained models with 25 probing tasks meant to reveal the specific skills that drive transfer. We observe that intermediate tasks requiring high-level inference and reasoning abilities tend to work best. We also observe that target task performance is strongly correlated with higher-level abilities such as coreference resolution. However, we fail to observe more granular correlations between probing and target task performance, highlighting the need for further work on broad-coverage probing benchmarks. We also observe evidence that the forgetting of knowledge learned during pretraining may limit our analysis, highlighting the need for further work on transfer learning methods in these settings.",
}


@article{pruksachatkun2020intermediate,
  title={Intermediate-Task Transfer Learning with Pretrained Models for Natural Language Understanding: When and Why Does It Work?},
  author={Pruksachatkun, Yada and Phang, Jason and Liu, Haokun and Htut, Phu Mon and Zhang, Xiaoyi and Pang, Richard Yuanzhe and Vania, Clara and Kann, Katharina and Bowman, Samuel R},
  journal={arXiv preprint arXiv:2005.00628},
  year={2020}
}