@inproceedings{ChizhovArnettKorotkovaetal.2024,
  author    = {Chizhov, Pavel and Arnett, Catherine and Korotkova, Elizaveta and Yamshchikov, Ivan P.},
  title     = {BPE Gets Picky: Efficient Vocabulary Refinement During Tokenizer Training},
  booktitle = {Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing},
  publisher = {Association for Computational Linguistics},
  address   = {Stroudsburg, PA, USA},
  doi       = {10.18653/v1/2024.emnlp-main.925},
  url       = {https://nbn-resolving.org/urn:nbn:de:bvb:863-opus-62831},
  institution = {Center for Artificial Intelligence (CAIRO)},
  pages     = {16587 -- 16604},
  year      = {2024},
  abstract  = {Language models can greatly benefit from efficient tokenization. However, they still mostly utilize the classical Byte-Pair Encoding (BPE) algorithm, a simple and reliable method. BPE has been shown to cause such issues as under-trained tokens and sub-optimal compression that may affect the downstream performance. We introduce PickyBPE, a modified BPE algorithm that carries out vocabulary refinement during tokenizer training by removing merges that leave intermediate "junk" tokens. Our method improves vocabulary efficiency, eliminates under-trained tokens, and does not compromise text compression. Our experiments show that this method either improves downstream performance or does not harm it.},
  language  = {en}
}