.. role:: hidden
    :class: hidden-section
.. currentmodule:: torchtune.modules.transforms.tokenizers


BaseTokenizer
=============

.. autoclass:: BaseTokenizer
    :members: