@inproceedings{hou-etal-2022-token,
    title = "Token Dropping for Efficient {BERT} Pretraining",
    author = "Hou, Le  and
      Pang, Richard Yuanzhe  and
      Zhou, Tianyi  and
      Wu, Yuexin  and
      Song, Xinying  and
      Song, Xiaodan  and
      Zhou, Denny",
    booktitle = "Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
    month = may,
    year = "2022",
    address = "Dublin, Ireland",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.acl-long.262",
    pages = "3774--3784",
    abstract = "Transformer-based models generally allocate the same amount of computation for each token in a given sequence. We develop a simple but effective {``}token dropping{''} method to accelerate the pretraining of transformer models, such as BERT, without degrading its performance on downstream tasks. In particular, we drop unimportant tokens starting from an intermediate layer in the model to make the model focus on important tokens more efficiently if with limited computational resource. The dropped tokens are later picked up by the last layer of the model so that the model still produces full-length sequences. We leverage the already built-in masked language modeling (MLM) loss to identify unimportant tokens with practically no computational overhead. In our experiments, this simple approach reduces the pretraining cost of BERT by 25{\%} while achieving similar overall fine-tuning performance on standard downstream tasks.",
}


@article{hou2022token,
  title={Token Dropping for Efficient BERT Pretraining},
  author={Hou, Le and Pang, Richard Yuanzhe and Zhou, Tianyi and Wu, Yuexin and Song, Xinying and Song, Xiaodan and Zhou, Denny},
  journal={arXiv preprint arXiv:2203.13240},
  year={2022}
}