Source code for torchtext.datasets.squad1

import os
from typing import Union, Tuple

from torchtext._internal.module_utils import is_module_available
from import (

if is_module_available("torchdata"):
    from torchdata.datapipes.iter import FileOpener, HttpReader, IterableWrapper

URL = {
    "train": "",
    "dev": "",

MD5 = {
    "train": "981b29407e0affa3b1b156f72073b945",
    "dev": "3e85deb501d4e538b6bc56f786231552",

    "train": 87599,
    "dev": 10570,


[docs]@_create_dataset_directory(dataset_name=DATASET_NAME) @_wrap_split_argument(("train", "dev")) def SQuAD1(root: str, split: Union[Tuple[str], str]): """SQuAD1 Dataset For additional details refer to Number of lines per split: - train: 87599 - dev: 10570 Args: root: Directory where the datasets are saved. Default: os.path.expanduser('~/.torchtext/cache') split: split or splits to be returned. Can be a string or tuple of strings. Default: (`train`, `dev`) :returns: DataPipe that yields data points from SQuaAD1 dataset which consist of context, question, list of answers and corresponding index in context :rtype: (str, str, list(str), list(int)) """ if not is_module_available("torchdata"): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at ``" ) url_dp = IterableWrapper([URL[split]]) # cache data on-disk with sanity check cache_dp = url_dp.on_disk_cache( filepath_fn=lambda x: os.path.join(root, os.path.basename(x)), hash_dict={os.path.join(root, os.path.basename(URL[split])): MD5[split]}, hash_type="md5", ) cache_dp = HttpReader(cache_dp).end_caching(mode="wb", same_filepath_fn=True) cache_dp = FileOpener(cache_dp, mode="b") return cache_dp.parse_json_files().read_squad()


Access comprehensive developer documentation for PyTorch

View Docs


Get in-depth tutorials for beginners and advanced developers

View Tutorials


Find development resources and get your questions answered

View Resources