# -*- coding: utf-8 -*-
"""
Preprocess custom text dataset using Torchtext
===============================================
**Author**: `Anupam Sharma `_
This tutorial illustrates the usage of torchtext on a dataset that is not built-in. In the tutorial,
we will preprocess a dataset that can be further utilized to train a sequence-to-sequence
model for machine translation (something like, in this tutorial: `Sequence to Sequence Learning
with Neural Networks `_) but without using legacy version
of torchtext.
In this tutorial, we will learn how to:
* Read a dataset
* Tokenize sentence
* Apply transforms to sentence
* Perform bucket batching
Let us assume that we need to prepare a dataset to train a model that can perform English to
German translation. We will use a tab-delimited German - English sentence pairs provided by
the `Tatoeba Project `_ which can be downloaded from
`this link `__.
Sentence pairs for other languages can be found in `this link `\
__.
"""
# %%
# Setup
# -----
#
# First, download the dataset, extract the zip, and note the path to the file `deu.txt`.
#
# Ensure that following packages are installed:
#
# * `Torchdata 0.6.0 `_ (`Installation instructions \
# `__)
# * `Torchtext 0.15.0 `_ (`Installation instructions \
# `__)
# * `Spacy `__
#
# Here, we are using `Spacy` to tokenize text. In simple words tokenization means to
# convert a sentence to list of words. Spacy is a python package used for various Natural
# Language Processing (NLP) tasks.
#
# Download the English and German models from Spacy as shown below:
#
# .. code-block:: shell
#
# python -m spacy download en_core_web_sm
# python -m spacy download de_core_news_sm
#
# %%
# Let us start by importing required modules:
import torchdata.datapipes as dp
import torchtext.transforms as T
import spacy
from torchtext.vocab import build_vocab_from_iterator
eng = spacy.load("en_core_web_sm") # Load the English model to tokenize English text
de = spacy.load("de_core_news_sm") # Load the German model to tokenize German text
# %%
# Now we will load the dataset
FILE_PATH = 'data/deu.txt'
data_pipe = dp.iter.IterableWrapper([FILE_PATH])
data_pipe = dp.iter.FileOpener(data_pipe, mode='rb')
data_pipe = data_pipe.parse_csv(skip_lines=0, delimiter='\t', as_tuple=True)
# %%
# In the above code block, we are doing following things:
#
# 1. At line 2, we are creating an iterable of filenames
# 2. At line 3, we pass the iterable to `FileOpener` which then
# opens the file in read mode
# 3. At line 4, we call a function to parse the file, which
# again returns an iterable of tuples representing each rows
# of the tab-delimited file
#
# DataPipes can be thought of something like a dataset object, on which
# we can perform various operations.
# Check `this tutorial `_ for more details on
# DataPipes.
#
# We can verify if the iterable has the pair of sentences as shown
# below:
for sample in data_pipe:
print(sample)
break
# %%
# Note that we also have attribution details along with pair of sentences. We will
# write a small function to remove the attribution details:
def removeAttribution(row):
"""
Function to keep the first two elements in a tuple
"""
return row[:2]
data_pipe = data_pipe.map(removeAttribution)
# %%
# The `map` function at line 6 in above code block can be used to apply some function
# on each elements of `data_pipe`. Now, we can verify that the `data_pipe` only contains
# pair of sentences.
for sample in data_pipe:
print(sample)
break
# %%
# Now, let us define few functions to perform tokenization:
def engTokenize(text):
"""
Tokenize an English text and return a list of tokens
"""
return [token.text for token in eng.tokenizer(text)]
def deTokenize(text):
"""
Tokenize a German text and return a list of tokens
"""
return [token.text for token in de.tokenizer(text)]
# %%
# Above function accepts a text and returns a list of words
# as shown below:
print(engTokenize("Have a good day!!!"))
print(deTokenize("Haben Sie einen guten Tag!!!"))
# %%
# Building the vocabulary
# -----------------------
# Let us consider an English sentence as the source and a German sentence as the target.
#
# Vocabulary can be considered as the set of unique words we have in the dataset.
# We will build vocabulary for both our source and target now.
#
# Let us define a function to get tokens from elements of tuples in the iterator.
def getTokens(data_iter, place):
"""
Function to yield tokens from an iterator. Since, our iterator contains
tuple of sentences (source and target), `place` parameters defines for which
index to return the tokens for. `place=0` for source and `place=1` for target
"""
for english, german in data_iter:
if place == 0:
yield engTokenize(english)
else:
yield deTokenize(german)
# %%
# Now, we will build vocabulary for source:
source_vocab = build_vocab_from_iterator(
getTokens(data_pipe,0),
min_freq=2,
specials= ['', '', '', ''],
special_first=True
)
source_vocab.set_default_index(source_vocab[''])
# %%
# The code above, builds the vocabulary from the iterator. In the above code block:
#
# * At line 2, we call the `getTokens()` function with `place=0` as we need vocabulary for
# source sentences.
# * At line 3, we set `min_freq=2`. This means, the function will skip those words that occurs
# less than 2 times.
# * At line 4, we specify some special tokens:
#
# * `` for start of sentence
# * `` for end of sentence
# * `` for unknown words. An example of unknown word is the one skipped because of
# `min_freq=2`.
# * `` is the padding token. While training, a model we mostly train in batches. In a
# batch, there can be sentences of different length. So, we pad the shorter sentences with
# `` token to make length of all sequences in the batch equal.
#
# * At line 5, we set `special_first=True`. Which means `` will get index 0, `` index 1,
# `` index 2, and will get index 3 in the vocabulary.
# * At line 7, we set default index as index of ``. That means if some word is not in
# vocabulary, we will use `` instead of that unknown word.
#
# Similarly, we will build vocabulary for target sentences:
target_vocab = build_vocab_from_iterator(
getTokens(data_pipe,1),
min_freq=2,
specials= ['', '', '', ''],
special_first=True
)
target_vocab.set_default_index(target_vocab[''])
# %%
# Note that the example above shows how can we add special tokens to our vocabulary. The
# special tokens may change based on the requirements.
#
# Now, we can verify that special tokens are placed at the beginning and then other words.
# In the below code, `source_vocab.get_itos()` returns a list with tokens at index based on
# vocabulary.
print(source_vocab.get_itos()[:9])
# %%
# Numericalize sentences using vocabulary
# ---------------------------------------
# After building the vocabulary, we need to convert our sentences to corresponding indices.
# Let us define some functions for this:
def getTransform(vocab):
"""
Create transforms based on given vocabulary. The returned transform is applied to sequence
of tokens.
"""
text_tranform = T.Sequential(
## converts the sentences to indices based on given vocabulary
T.VocabTransform(vocab=vocab),
## Add at beginning of each sentence. 1 because the index for in vocabulary is
# 1 as seen in previous section
T.AddToken(1, begin=True),
## Add at beginning of each sentence. 2 because the index for in vocabulary is
# 2 as seen in previous section
T.AddToken(2, begin=False)
)
return text_tranform
# %%
# Now, let us see how to use the above function. The function returns an object of `Transforms`
# which we will use on our sentence. Let us take a random sentence and check how the transform
# works.
temp_list = list(data_pipe)
some_sentence = temp_list[798][0]
print("Some sentence=", end="")
print(some_sentence)
transformed_sentence = getTransform(source_vocab)(engTokenize(some_sentence))
print("Transformed sentence=", end="")
print(transformed_sentence)
index_to_string = source_vocab.get_itos()
for index in transformed_sentence:
print(index_to_string[index], end=" ")
# %%
# In the above code,:
#
# * At line 2, we take a source sentence from list that we created from `data_pipe` at line 1
# * At line 5, we get a transform based on a source vocabulary and apply it to a tokenized
# sentence. Note that transforms take list of words and not a sentence.
# * At line 8, we get the mapping of index to string and then use it get the transformed
# sentence
#
# Now we will use DataPipe functions to apply transform to all our sentences.
# Let us define some more functions for this.
def applyTransform(sequence_pair):
"""
Apply transforms to sequence of tokens in a sequence pair
"""
return (
getTransform(source_vocab)(engTokenize(sequence_pair[0])),
getTransform(target_vocab)(deTokenize(sequence_pair[1]))
)
data_pipe = data_pipe.map(applyTransform) ## Apply the function to each element in the iterator
temp_list = list(data_pipe)
print(temp_list[0])
# %%
# Make batches (with bucket batch)
# --------------------------------
# Generally, we train models in batches. While working for sequence to sequence models, it is
# recommended to keep the length of sequences in a batch similar. For that we will use
# `bucketbatch` function of `data_pipe`.
#
# Let us define some functions that will be used by the `bucketbatch` function.
def sortBucket(bucket):
"""
Function to sort a given bucket. Here, we want to sort based on the length of
source and target sequence.
"""
return sorted(bucket, key=lambda x: (len(x[0]), len(x[1])))
# %%
# Now, we will apply the `bucketbatch` function:
data_pipe = data_pipe.bucketbatch(
batch_size = 4, batch_num=5, bucket_num=1,
use_in_batch_shuffle=False, sort_key=sortBucket
)
# %%
# In the above code block:
#
# * We keep batch size = 4.
# * `batch_num` is the number of batches to keep in a bucket
# * `bucket_num` is the number of buckets to keep in a pool for shuffling
# * `sort_key` specifies the function that takes a bucket and sorts it
#
# Now, let us consider a batch of source sentences as `X` and a batch of target sentences as `y`.
# Generally, while training a model, we predict on a batch of `X` and compare the result with `y`.
# But, a batch in our `data_pipe` is of the form `[(X_1,y_1), (X_2,y_2), (X_3,y_3), (X_4,y_4)]`:
print(list(data_pipe)[0])
# %%
# So, we will now convert them into the form: `((X_1,X_2,X_3,X_4), (y_1,y_2,y_3,y_4))`.
# For this we will write a small function:
def separateSourceTarget(sequence_pairs):
"""
input of form: `[(X_1,y_1), (X_2,y_2), (X_3,y_3), (X_4,y_4)]`
output of form: `((X_1,X_2,X_3,X_4), (y_1,y_2,y_3,y_4))`
"""
sources,targets = zip(*sequence_pairs)
return sources,targets
## Apply the function to each element in the iterator
data_pipe = data_pipe.map(separateSourceTarget)
print(list(data_pipe)[0])
# %%
# Now, we have the data as desired.
#
# Padding
# -------
# As discussed earlier while building vocabulary, we need to pad shorter sentences in a batch to
# make all the sequences in a batch of equal length. We can perform padding as follows:
def applyPadding(pair_of_sequences):
"""
Convert sequences to tensors and apply padding
"""
return (T.ToTensor(0)(list(pair_of_sequences[0])), T.ToTensor(0)(list(pair_of_sequences[1])))
## `T.ToTensor(0)` returns a transform that converts the sequence to `torch.tensor` and also applies
# padding. Here, `0` is passed to the constructor to specify the index of the `` token in the
# vocabulary.
data_pipe = data_pipe.map(applyPadding)
# %%
# Now, we can use the index to string mapping to see how the sequence would look with tokens
# instead of indices:
source_index_to_string = source_vocab.get_itos()
target_index_to_string = target_vocab.get_itos()
def showSomeTransformedSentences(data_pipe):
"""
Function to show how the sentences look like after applying all transforms.
Here we try to print actual words instead of corresponding index
"""
for sources,targets in data_pipe:
if sources[0][-1] != 0:
continue # Just to visualize padding of shorter sentences
for i in range(4):
source = ""
for token in sources[i]:
source += " " + source_index_to_string[token]
target = ""
for token in targets[i]:
target += " " + target_index_to_string[token]
print(f"Source: {source}")
print(f"Traget: {target}")
break
showSomeTransformedSentences(data_pipe)
# %%
# In the above output we can observe that the shorter sentences are padded with ``. Now, we
# can use `data_pipe` while writing our training function.
#
# Some parts of this tutorial was inspired from `this article
# `__.