Shortcuts

Source code for torchtext.datasets.penntreebank

import logging
from torchtext.utils import download_from_url
from torchtext.data.datasets_utils import _RawTextIterableDataset
from torchtext.data.datasets_utils import _wrap_split_argument
from torchtext.data.datasets_utils import _add_docstring_header
import io

URL = {
    'train': "https://raw.githubusercontent.com/wojzaremba/lstm/master/data/ptb.train.txt",
    'test': "https://raw.githubusercontent.com/wojzaremba/lstm/master/data/ptb.test.txt",
    'valid': "https://raw.githubusercontent.com/wojzaremba/lstm/master/data/ptb.valid.txt",
}

MD5 = {
    'train': "f26c4b92c5fdc7b3f8c7cdcb991d8420",
    'valid': "aa0affc06ff7c36e977d7cd49e3839bf",
    'test': "8b80168b89c18661a38ef683c0dc3721",
}

NUM_LINES = {
    'train': 42068,
    'valid': 3370,
    'test': 3761,
}


[docs]@_add_docstring_header(num_lines=NUM_LINES) @_wrap_split_argument(('train', 'valid', 'test')) def PennTreebank(root, split): path = download_from_url(URL[split], root=root, hash_value=MD5[split], hash_type='md5') logging.info('Creating {} data'.format(split)) return _RawTextIterableDataset('PennTreebank', NUM_LINES[split], iter(io.open(path, encoding="utf8")))

Docs

Access comprehensive developer documentation for PyTorch

View Docs

Tutorials

Get in-depth tutorials for beginners and advanced developers

View Tutorials

Resources

Find development resources and get your questions answered

View Resources