Safe lab preview

Text Representation Py Torch

This is a sanitized, read-only preview. Nothing executes in this page.

Read-only

Notebook preview

Text Representation Py Torch

Download notebook

Markdown cell[1]

> **Integrated runtime note:** This preview uses a small deterministic, rights-safe offline fixture for reproducible learning. Full-scale results require the lesson's documented dataset or model in an approved external environment. # Text classification task We will study a compact text-classification task using a small news corpus written specifically for this edition. Each item belongs to one of four categories: World, Sports, Business, or Sci/Tech. ## The dataset The dataset is embedded in the notebook so the exercise is deterministic, quick to run, and independent of retired data libraries or network downloads.

Code cell[2]

import collections
import random
import re

import numpy as np
import torch

# course-edition deterministic NLP utilities v2

COURSE_NEWS_CLASSES = ["World", "Sports", "Business", "Sci/Tech"]

# This compact corpus was written for this edition. It keeps the notebooks
# deterministic, network-free, and quick enough to run on a learner's CPU.
COURSE_NEWS_TRAIN = [
    (1, "Paris hosted a regional summit where diplomats discussed water security, peaceful trade routes, and a shared emergency plan for neighboring communities."),
    (1, "The elected council approved a cross-border health agreement after delegates reviewed hospital capacity, medicine access, and transparent public reporting."),
    (1, "A coastal city welcomed observers for a public vote, while local groups published clear guidance about polling places and accessible transportation."),
    (1, "The king and queen met a woman who leads the relief agency and a man who coordinates volunteers, then the group announced a neutral humanitarian corridor."),
    (1, "Ministers signed a climate adaptation pledge that funds drought monitoring, resilient farms, and open scientific exchange across several countries."),
    (1, "Community mediators opened a week of talks focused on civilian safety, reliable food deliveries, and practical steps toward a lasting ceasefire."),
    (2, "The basketball team completed a patient comeback in the final quarter, using quick passes, strong defense, and a balanced scoring plan to win the tournament."),
    (2, "A young runner broke the course record after months of careful training, recovery sessions, nutrition planning, and support from her local athletics club."),
    (2, "The football coach praised disciplined play after the squad protected an early lead and created chances through accurate movement on both wings."),
    (2, "Fans filled the arena for a close volleyball final in which both teams served aggressively, defended long rallies, and respected every referee decision."),
    (2, "The tennis champion returned after injury and won a demanding match by varying pace, placing serves carefully, and staying calm during two tie breaks."),
    (2, "Organizers added inclusive swimming events to the city games so more athletes can compete with safe facilities, trained officials, and fair timing equipment."),
    (3, "Microsoft announced a small-business program that combines cloud credits, security workshops, and practical accounting support for new local companies."),
    (3, "Global funds moved cautiously after the central bank held interest rates steady and asked lenders to publish clearer information about consumer borrowing costs."),
    (3, "A neighborhood market expanded its delivery service, hired twelve workers, and invested revenue in reusable packaging supplied by another local business."),
    (3, "The manufacturer reported stable quarterly sales while noting that shipping delays and higher material prices could reduce margins later in the year."),
    (3, "Two payment companies agreed to share fraud signals through a privacy-preserving system designed to protect customers without exposing purchase details."),
    (3, "A cooperative offered farmers transparent contracts and faster invoices, helping members plan equipment repairs before the next growing season begins."),
    (4, "Researchers trained a compact neural network to identify damaged solar panels, then published the evaluation data and documented where the model still fails."),
    (4, "A university technology lab released an open sensor design that measures classroom air quality and stores only anonymous readings for public analysis."),
    (4, "Engineers improved a battery recycling process by recovering more useful minerals at lower temperatures, reducing energy use during the pilot study."),
    (4, "The space telescope captured a detailed spectrum from a distant planet, giving scientists new evidence about clouds and molecules in its atmosphere."),
    (4, "A software team tested an accessibility assistant with keyboard users and screen-reader experts before changing the interface and publishing the results."),
    (4, "Students built a small robot that maps indoor obstacles with inexpensive sensors, explains each route choice, and says hello when a test run begins."),
]

COURSE_NEWS_TEST = [
    (1, "Regional delegates published a joint disaster response schedule after reviewing evacuation routes and communication gaps."),
    (1, "Election monitors confirmed the final count and recommended clearer access rules for voters who need assistance."),
    (2, "The basketball captain scored late, but credited the victory to defense, passing, and careful preparation across the whole season."),
    (2, "A cycling club opened a safe youth race with trained marshals, marked turns, and free equipment checks."),
    (3, "Technology shares rose while retail funds remained cautious after companies issued mixed forecasts for the next quarter."),
    (3, "The family business secured a modest loan to replace old equipment and expand its apprenticeship program."),
    (4, "A research team released a smaller language model with documented energy measurements and a public evaluation set."),
    (4, "Scientists used open satellite data to improve flood warnings and shared the software with local emergency teams."),
]

def course_tokenize(text):
    return re.findall(r"[a-z0-9]+(?:'[a-z0-9]+)?", str(text).lower())

def course_ngrams_iterator(tokens, ngrams=1):
    tokens = list(tokens)
    for token in tokens:
        yield token
    for size in range(2, max(1, ngrams) + 1):
        for start in range(0, len(tokens) - size + 1):
            yield " ".join(tokens[start:start + size])

class CourseVocabulary:
    def __init__(self, counter, min_freq=1, max_tokens=None):
        ordered = sorted(
            (token for token, count in counter.items() if count >= min_freq),
            key=lambda token: (-counter[token], token),
        )
        if max_tokens is not None:
            ordered = ordered[:max(0, max_tokens - 1)]
        self.itos = ["<unk>"] + [token for token in ordered if token != "<unk>"]
        self.stoi = {token: index for index, token in enumerate(self.itos)}

    def __len__(self):
        return len(self.itos)

    def __getitem__(self, token):
        return self.stoi.get(token, 0)

    def get_stoi(self):
        return dict(self.stoi)

    def get_itos(self):
        return list(self.itos)

def build_course_vocab(dataset, ngrams=1, min_freq=1, max_tokens=None, tokenizer=course_tokenize):
    counter = collections.Counter()
    for _label, text in dataset:
        counter.update(course_ngrams_iterator(tokenizer(text), ngrams=ngrams))
    return CourseVocabulary(counter, min_freq=min_freq, max_tokens=max_tokens)

def load_course_fixture(ngrams=1, min_freq=1, vocab_size=None, lines_cnt=None):
    global vocab, tokenizer
    tokenizer = course_tokenize
    train_dataset = list(COURSE_NEWS_TRAIN)
    test_dataset = list(COURSE_NEWS_TEST)
    vocabulary_rows = train_dataset if lines_cnt is None else train_dataset[:lines_cnt]
    vocab = build_course_vocab(
        vocabulary_rows,
        ngrams=ngrams,
        min_freq=min_freq,
        max_tokens=vocab_size,
        tokenizer=tokenizer,
    )
    return train_dataset, test_dataset, list(COURSE_NEWS_CLASSES), vocab

def encode(text, voc=None, unk=0, tokenizer=course_tokenize):
    selected_vocab = vocab if voc is None else voc
    stoi = selected_vocab.get_stoi()
    return [stoi.get(token, unk) for token in tokenizer(text)]

def train_epoch(net, dataloader, lr=0.01, optimizer=None, loss_fn=None, epoch_size=None, report_freq=200):
    optimizer = optimizer or torch.optim.Adam(net.parameters(), lr=lr)
    loss_fn = (loss_fn or torch.nn.CrossEntropyLoss()).to(device)
    net.train()
    total_loss, accuracy, count, batch_index = 0.0, 0, 0, 0
    for labels, features in dataloader:
        optimizer.zero_grad()
        features, labels = features.to(device), labels.to(device)
        output = net(features)
        loss = loss_fn(output, labels)
        loss.backward()
        optimizer.step()
        total_loss += loss.item()
        accuracy += (output.argmax(1) == labels).sum().item()
        count += len(labels)
        batch_index += 1
        if batch_index % report_freq == 0:
            print(f"{count}: acc={accuracy / count:.4f}")
        if epoch_size and count > epoch_size:
            break
    return total_loss / max(count, 1), accuracy / max(count, 1)

def padify(batch, voc=None, tokenizer=course_tokenize):
    vectors = [encode(item[1], voc=voc, tokenizer=tokenizer) for item in batch]
    max_length = max(map(len, vectors))
    return (
        torch.LongTensor([item[0] - 1 for item in batch]),
        torch.stack([
            torch.nn.functional.pad(torch.tensor(vector), (0, max_length - len(vector)), value=0)
            for vector in vectors
        ]),
    )

def offsetify(batch, voc=None):
    vectors = [torch.tensor(encode(item[1], voc=voc)) for item in batch]
    offsets = torch.tensor(([0] + [len(vector) for vector in vectors])[:-1]).cumsum(dim=0)
    return torch.LongTensor([item[0] - 1 for item in batch]), torch.cat(vectors), offsets

def train_epoch_emb(net, dataloader, lr=0.01, optimizer=None, loss_fn=None, epoch_size=None, report_freq=200, use_pack_sequence=False):
    optimizer = optimizer or torch.optim.Adam(net.parameters(), lr=lr)
    loss_fn = (loss_fn or torch.nn.CrossEntropyLoss()).to(device)
    net.train()
    total_loss, accuracy, count, batch_index = 0.0, 0, 0, 0
    for labels, text, offsets in dataloader:
        optimizer.zero_grad()
        labels, text = labels.to(device), text.to(device)
        offsets = offsets.to("cpu" if use_pack_sequence else device)
        output = net(text, offsets)
        loss = loss_fn(output, labels)
        loss.backward()
        optimizer.step()
        total_loss += loss.item()
        accuracy += (output.argmax(1) == labels).sum().item()
        count += len(labels)
        batch_index += 1
        if batch_index % report_freq == 0:
            print(f"{count}: acc={accuracy / count:.4f}")
        if epoch_size and count > epoch_size:
            break
    return total_loss / max(count, 1), accuracy / max(count, 1)

random.seed(42)
np.random.seed(42)
torch.manual_seed(42)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
tokenizer = course_tokenize
vocab = None

train_dataset, test_dataset, classes, vocab = load_course_fixture()

Markdown cell[3]

Here, `train_dataset` and `test_dataset` contain collections that return pairs of label (number of class) and text respectively, for example:

Code cell[4]

list(train_dataset)[0]

Markdown cell[5]

So, let's print out the first 10 new headlines from our dataset:

Code cell[6]

for i, x in zip(range(5), train_dataset):
    print(f"**{classes[x[0] - 1]}** -> {x[1]}")

Markdown cell[7]

Because datasets are iterators, if we want to use the data multiple times we need to convert it to list:

Code cell[8]

train_dataset, test_dataset, classes, vocab = load_course_fixture()

Markdown cell[9]

## Tokenization Now we need to convert text into **numbers** that can be represented as tensors. If we want word-level representation, we need to do two things: * use **tokenizer** to split text into **tokens** * build a **vocabulary** of those tokens.

Code cell[10]

tokenizer('He said: hello')

Code cell[11]

counter = collections.Counter()
for label, line in train_dataset:
    counter.update(tokenizer(line))
vocab = CourseVocabulary(counter, min_freq=1)

Markdown cell[12]

Using vocabulary, we can easily encode out tokenized string into a set of numbers:

Code cell[13]

vocab_size = len(vocab)
print(f"Vocab size is {vocab_size}")

stoi = vocab.get_stoi()

def encode(x):
    return [stoi.get(token, 0) for token in tokenizer(x)]

encode('I love to play with my words')

Markdown cell[14]

## Bag of Words text representation Because words represent meaning, sometimes we can figure out the meaning of a text by just looking at the individual words, regardless of their order in the sentence. For example, when classifying news, words like *weather*, *snow* are likely to indicate *weather forecast*, while words like *stocks*, *dollar* would count towards *financial news*. **Bag of Words** (BoW) vector representation is the most commonly used traditional vector representation. Each word is linked to a vector index, vector element contains the number of occurrences of a word in a given document. > **Figure description:** Image showing how a bag of words vector representation is represented in memory. > **Note**: You can also think of BoW as a sum of all one-hot-encoded vectors for individual words in the text. Below is an example of how to generate a bag of word representation using the Scikit Learn python library:

Code cell[15]

from sklearn.feature_extraction.text import CountVectorizer
vectorizer = CountVectorizer()
corpus = [
        'I like hot dogs.',
        'The dog ran fast.',
        'Its hot outside.',
    ]
vectorizer.fit_transform(corpus)
vectorizer.transform(['My dog likes hot dogs on a hot day.']).toarray()

Markdown cell[16]

To compute bag-of-words vector from the vector representation of our AG_NEWS dataset, we can use the following function:

Code cell[17]

vocab_size = len(vocab)

def to_bow(text,bow_vocab_size=vocab_size):
    res = torch.zeros(bow_vocab_size,dtype=torch.float32)
    for i in encode(text):
        if i<bow_vocab_size:
            res[i] += 1
    return res

print(to_bow(train_dataset[0][1]))

Markdown cell[18]

> **Note:** Here we are using global `vocab_size` variable to specify default size of the vocabulary. Since often vocabulary size is pretty big, we can limit the size of the vocabulary to most frequent words. Try lowering `vocab_size` value and running the code below, and see how it affects the accuracy. You should expect some accuracy drop, but not dramatic, in lieu of higher performance.

Markdown cell[19]

## Training BoW classifier Now that we have learned how to build Bag-of-Words representation of our text, let's train a classifier on top of it. First, we need to convert our dataset for training in such a way, that all positional vector representations are converted to bag-of-words representation. This can be achieved by passing `bowify` function as `collate_fn` parameter to standard torch `DataLoader`:

Code cell[20]

from torch.utils.data import DataLoader
import numpy as np

# this collate function gets list of batch_size tuples, and needs to
# return a pair of label-feature tensors for the whole minibatch
def bowify(b):
    return (
            torch.LongTensor([t[0]-1 for t in b]),
            torch.stack([to_bow(t[1]) for t in b])
    )

train_loader = DataLoader(train_dataset, batch_size=16, collate_fn=bowify, shuffle=True)
test_loader = DataLoader(test_dataset, batch_size=16, collate_fn=bowify, shuffle=True)

Markdown cell[21]

Now let's define a simple classifier neural network that contains one linear layer. The size of the input vector equals to `vocab_size`, and output size corresponds to the number of classes (4). Because we are solving classification task, the final activation function is `LogSoftmax`.

Code cell[22]

net = torch.nn.Sequential(torch.nn.Linear(vocab_size,4),torch.nn.LogSoftmax(dim=1))

Markdown cell[23]

Now we will define standard PyTorch training loop. Because our dataset is quite large, for our teaching purpose we will train only for one epoch, and sometimes even for less than an epoch (specifying the `epoch_size` parameter allows us to limit training). We would also report accumulated training accuracy during training; the frequency of reporting is specified using `report_freq` parameter.

Code cell[24]

def train_epoch(net,dataloader,lr=0.01,optimizer=None,loss_fn = torch.nn.NLLLoss(),epoch_size=None, report_freq=200):
    optimizer = optimizer or torch.optim.Adam(net.parameters(),lr=lr)
    net.train()
    total_loss,acc,count,i = 0,0,0,0
    for labels,features in dataloader:
        optimizer.zero_grad()
        out = net(features)
        loss = loss_fn(out,labels) #cross_entropy(out,labels)
        loss.backward()
        optimizer.step()
        total_loss+=loss
        _,predicted = torch.max(out,1)
        acc+=(predicted==labels).sum()
        count+=len(labels)
        i+=1
        if i%report_freq==0:
            print(f"{count}: acc={acc.item()/count}")
        if epoch_size and count>epoch_size:
            break
    return total_loss.item()/count, acc.item()/count

Code cell[25]

train_epoch(net,train_loader,epoch_size=15000)

Markdown cell[26]

## BiGrams, TriGrams and N-Grams One limitation of a bag of words approach is that some words are part of multi word expressions, for example, the word 'hot dog' has a completely different meaning than the words 'hot' and 'dog' in other contexts. If we represent words 'hot` and 'dog' always by the same vectors, it can confuse our model. To address this, **N-gram representations** are often used in methods of document classification, where the frequency of each word, bi-word or tri-word is a useful feature for training classifiers. In bigram representation, for example, we will add all word pairs to the vocabulary, in addition to original words. Below is an example of how to generate a bigram bag of word representation using the Scikit Learn:

Code cell[27]

bigram_vectorizer = CountVectorizer(ngram_range=(1, 2), token_pattern=r'\b\w+\b', min_df=1)
corpus = [
        'I like hot dogs.',
        'The dog ran fast.',
        'Its hot outside.',
    ]
bigram_vectorizer.fit_transform(corpus)
print("Vocabulary:\n",bigram_vectorizer.vocabulary_)
bigram_vectorizer.transform(['My dog likes hot dogs on a hot day.']).toarray()

Markdown cell[28]

The main drawback of N-gram approach is that vocabulary size starts to grow extremely fast. In practice, we need to combine N-gram representation with some dimensionality reduction techniques, such as *embeddings*, which we will discuss in the next unit. To use N-gram representation in our **AG News** dataset, we need to build special ngram vocabulary:

Code cell[29]

counter = collections.Counter()
for label, line in train_dataset:
    counter.update(course_ngrams_iterator(tokenizer(line), ngrams=2))

bi_vocab = CourseVocabulary(counter, min_freq=1)
print("Bigram vocabulary length = ", len(bi_vocab))

Markdown cell[30]

We could then use the same code as above to train the classifier, however, it would be very memory-inefficient. In the next unit, we will train bigram classifier using embeddings. > **Note:** You can only leave those ngrams that occur in the text more than specified number of times. This will make sure that infrequent bigrams will be omitted, and will decrease the dimensionality significantly. To do this, set `min_freq` parameter to a higher value, and observe the length of vocabulary change.

Markdown cell[31]

## Term Frequency Inverse Document Frequency TF-IDF In BoW representation, word occurrences are evenly weighted, regardless of the word itself. However, it is clear that frequent words, such as *a*, *in*, etc. are much less important for the classification, than specialized terms. In fact, in most NLP tasks some words are more relevant than others. **TF-IDF** stands for **term frequency–inverse document frequency**. It is a variation of bag of words, where instead of a binary 0/1 value indicating the appearance of a word in a document, a floating-point value is used, which is related to the frequency of word occurrence in the corpus. More formally, the weight $w_{ij}$ of a word $i$ in the document $j$ is defined as: $$ w_{ij} = tf_{ij}\times\log({N\over df_i}) $$ where * $tf_{ij}$ is the number of occurrences of $i$ in $j$, i.e. the BoW value we have seen before * $N$ is the number of documents in the collection * $df_i$ is the number of documents containing the word $i$ in the whole collection TF-IDF value $w_{ij}$ increases proportionally to the number of times a word appears in a document and is offset by the number of documents in the corpus that contains the word, which helps to adjust for the fact that some words appear more frequently than others. For example, if the word appears in *every* document in the collection, $df_i=N$, and $w_{ij}=0$, and those terms would be completely disregarded. You can easily create TF-IDF vectorization of text using Scikit Learn:

Code cell[32]

from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer(ngram_range=(1,2))
vectorizer.fit_transform(corpus)
vectorizer.transform(['My dog likes hot dogs on a hot day.']).toarray()

Markdown cell[33]

## Conclusion However even though TF-IDF representations provide frequency weight to different words they are unable to represent meaning or order. As the famous linguist J. R. Firth said in 1935, “The complete meaning of a word is always contextual, and no study of meaning apart from context can be taken seriously.”. We will learn later in the course how to capture contextual information from text using language modeling.

Outputs, execution counts, widgets, and active content were removed during import. Run notebooks only in an external environment you trust.

Record your practice

Optional self-reporting helps you remember what you practiced and never gates course completion.

Back to lesson

import collections import random import re import numpy as np import torch # course-edition deterministic NLP utilities v2 COURSE_NEWS_CLASSES = ["World", "Sports", "Business", "Sci/Tech"] # This compact corpus was written for this edition. It keeps the notebooks # deterministic, network-free, and quick enough to run on a learner's CPU. COURSE_NEWS_TRAIN = [ (1, "Paris hosted a regional summit where diplomats discussed water security, peaceful trade routes, and a shared emergency plan for neighboring communities."), (1, "The elected council approved a cross-border health agreement after delegates reviewed hospital capacity, medicine access, and transparent public reporting."), (1, "A coastal city welcomed observers for a public vote, while local groups published clear guidance about polling places and accessible transportation."), (1, "The king and queen met a woman who leads the relief agency and a man who coordinates volunteers, then the group announced a neutral humanitarian corridor."), (1, "Ministers signed a climate adaptation pledge that funds drought monitoring, resilient farms, and open scientific exchange across several countries."), (1, "Community mediators opened a week of talks focused on civilian safety, reliable food deliveries, and practical steps toward a lasting ceasefire."), (2, "The basketball team completed a patient comeback in the final quarter, using quick passes, strong defense, and a balanced scoring plan to win the tournament."), (2, "A young runner broke the course record after months of careful training, recovery sessions, nutrition planning, and support from her local athletics club."), (2, "The football coach praised disciplined play after the squad protected an early lead and created chances through accurate movement on both wings."), (2, "Fans filled the arena for a close volleyball final in which both teams served aggressively, defended long rallies, and respected every referee decision."), (2, "The tennis champion returned after injury and won a demanding match by varying pace, placing serves carefully, and staying calm during two tie breaks."), (2, "Organizers added inclusive swimming events to the city games so more athletes can compete with safe facilities, trained officials, and fair timing equipment."), (3, "Microsoft announced a small-business program that combines cloud credits, security workshops, and practical accounting support for new local companies."), (3, "Global funds moved cautiously after the central bank held interest rates steady and asked lenders to publish clearer information about consumer borrowing costs."), (3, "A neighborhood market expanded its delivery service, hired twelve workers, and invested revenue in reusable packaging supplied by another local business."), (3, "The manufacturer reported stable quarterly sales while noting that shipping delays and higher material prices could reduce margins later in the year."), (3, "Two payment companies agreed to share fraud signals through a privacy-preserving system designed to protect customers without exposing purchase details."), (3, "A cooperative offered farmers transparent contracts and faster invoices, helping members plan equipment repairs before the next growing season begins."), (4, "Researchers trained a compact neural network to identify damaged solar panels, then published the evaluation data and documented where the model still fails."), (4, "A university technology lab released an open sensor design that measures classroom air quality and stores only anonymous readings for public analysis."), (4, "Engineers improved a battery recycling process by recovering more useful minerals at lower temperatures, reducing energy use during the pilot study."), (4, "The space telescope captured a detailed spectrum from a distant planet, giving scientists new evidence about clouds and molecules in its atmosphere."), (4, "A software team tested an accessibility assistant with keyboard users and screen-reader experts before changing the interface and publishing the results."), (4, "Students built a small robot that maps indoor obstacles with inexpensive sensors, explains each route choice, and says hello when a test run begins."), ] COURSE_NEWS_TEST = [ (1, "Regional delegates published a joint disaster response schedule after reviewing evacuation routes and communication gaps."), (1, "Election monitors confirmed the final count and recommended clearer access rules for voters who need assistance."), (2, "The basketball captain scored late, but credited the victory to defense, passing, and careful preparation across the whole season."), (2, "A cycling club opened a safe youth race with trained marshals, marked turns, and free equipment checks."), (3, "Technology shares rose while retail funds remained cautious after companies issued mixed forecasts for the next quarter."), (3, "The family business secured a modest loan to replace old equipment and expand its apprenticeship program."), (4, "A research team released a smaller language model with documented energy measurements and a public evaluation set."), (4, "Scientists used open satellite data to improve flood warnings and shared the software with local emergency teams."), ] def course_tokenize(text): return re.findall(r"[a-z0-9]+(?:'[a-z0-9]+)?", str(text).lower()) def course_ngrams_iterator(tokens, ngrams=1): tokens = list(tokens) for token in tokens: yield token for size in range(2, max(1, ngrams) + 1): for start in range(0, len(tokens) - size + 1): yield " ".join(tokens[start:start + size]) class CourseVocabulary: def __init__(self, counter, min_freq=1, max_tokens=None): ordered = sorted( (token for token, count in counter.items() if count >= min_freq), key=lambda token: (-counter[token], token), ) if max_tokens is not None: ordered = ordered[:max(0, max_tokens - 1)] self.itos = ["<unk>"] + [token for token in ordered if token != "<unk>"] self.stoi = {token: index for index, token in enumerate(self.itos)} def __len__(self): return len(self.itos) def __getitem__(self, token): return self.stoi.get(token, 0) def get_stoi(self): return dict(self.stoi) def get_itos(self): return list(self.itos) def build_course_vocab(dataset, ngrams=1, min_freq=1, max_tokens=None, tokenizer=course_tokenize): counter = collections.Counter() for _label, text in dataset: counter.update(course_ngrams_iterator(tokenizer(text), ngrams=ngrams)) return CourseVocabulary(counter, min_freq=min_freq, max_tokens=max_tokens) def load_course_fixture(ngrams=1, min_freq=1, vocab_size=None, lines_cnt=None): global vocab, tokenizer tokenizer = course_tokenize train_dataset = list(COURSE_NEWS_TRAIN) test_dataset = list(COURSE_NEWS_TEST) vocabulary_rows = train_dataset if lines_cnt is None else train_dataset[:lines_cnt] vocab = build_course_vocab( vocabulary_rows, ngrams=ngrams, min_freq=min_freq, max_tokens=vocab_size, tokenizer=tokenizer, ) return train_dataset, test_dataset, list(COURSE_NEWS_CLASSES), vocab def encode(text, voc=None, unk=0, tokenizer=course_tokenize): selected_vocab = vocab if voc is None else voc stoi = selected_vocab.get_stoi() return [stoi.get(token, unk) for token in tokenizer(text)] def train_epoch(net, dataloader, lr=0.01, optimizer=None, loss_fn=None, epoch_size=None, report_freq=200): optimizer = optimizer or torch.optim.Adam(net.parameters(), lr=lr) loss_fn = (loss_fn or torch.nn.CrossEntropyLoss()).to(device) net.train() total_loss, accuracy, count, batch_index = 0.0, 0, 0, 0 for labels, features in dataloader: optimizer.zero_grad() features, labels = features.to(device), labels.to(device) output = net(features) loss = loss_fn(output, labels) loss.backward() optimizer.step() total_loss += loss.item() accuracy += (output.argmax(1) == labels).sum().item() count += len(labels) batch_index += 1 if batch_index % report_freq == 0: print(f"{count}: acc={accuracy / count:.4f}") if epoch_size and count > epoch_size: break return total_loss / max(count, 1), accuracy / max(count, 1) def padify(batch, voc=None, tokenizer=course_tokenize): vectors = [encode(item[1], voc=voc, tokenizer=tokenizer) for item in batch] max_length = max(map(len, vectors)) return ( torch.LongTensor([item[0] - 1 for item in batch]), torch.stack([ torch.nn.functional.pad(torch.tensor(vector), (0, max_length - len(vector)), value=0) for vector in vectors ]), ) def offsetify(batch, voc=None): vectors = [torch.tensor(encode(item[1], voc=voc)) for item in batch] offsets = torch.tensor(([0] + [len(vector) for vector in vectors])[:-1]).cumsum(dim=0) return torch.LongTensor([item[0] - 1 for item in batch]), torch.cat(vectors), offsets def train_epoch_emb(net, dataloader, lr=0.01, optimizer=None, loss_fn=None, epoch_size=None, report_freq=200, use_pack_sequence=False): optimizer = optimizer or torch.optim.Adam(net.parameters(), lr=lr) loss_fn = (loss_fn or torch.nn.CrossEntropyLoss()).to(device) net.train() total_loss, accuracy, count, batch_index = 0.0, 0, 0, 0 for labels, text, offsets in dataloader: optimizer.zero_grad() labels, text = labels.to(device), text.to(device) offsets = offsets.to("cpu" if use_pack_sequence else device) output = net(text, offsets) loss = loss_fn(output, labels) loss.backward() optimizer.step() total_loss += loss.item() accuracy += (output.argmax(1) == labels).sum().item() count += len(labels) batch_index += 1 if batch_index % report_freq == 0: print(f"{count}: acc={accuracy / count:.4f}") if epoch_size and count > epoch_size: break return total_loss / max(count, 1), accuracy / max(count, 1) random.seed(42) np.random.seed(42) torch.manual_seed(42) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") tokenizer = course_tokenize vocab = None train_dataset, test_dataset, classes, vocab = load_course_fixture()

from sklearn.feature_extraction.text import CountVectorizer vectorizer = CountVectorizer() corpus = [ 'I like hot dogs.', 'The dog ran fast.', 'Its hot outside.', ] vectorizer.fit_transform(corpus) vectorizer.transform(['My dog likes hot dogs on a hot day.']).toarray()

vocab_size = len(vocab) def to_bow(text,bow_vocab_size=vocab_size): res = torch.zeros(bow_vocab_size,dtype=torch.float32) for i in encode(text): if i<bow_vocab_size: res[i] += 1 return res print(to_bow(train_dataset[0][1]))

from torch.utils.data import DataLoader import numpy as np # this collate function gets list of batch_size tuples, and needs to # return a pair of label-feature tensors for the whole minibatch def bowify(b): return ( torch.LongTensor([t[0]-1 for t in b]), torch.stack([to_bow(t[1]) for t in b]) ) train_loader = DataLoader(train_dataset, batch_size=16, collate_fn=bowify, shuffle=True) test_loader = DataLoader(test_dataset, batch_size=16, collate_fn=bowify, shuffle=True)

def train_epoch(net,dataloader,lr=0.01,optimizer=None,loss_fn = torch.nn.NLLLoss(),epoch_size=None, report_freq=200): optimizer = optimizer or torch.optim.Adam(net.parameters(),lr=lr) net.train() total_loss,acc,count,i = 0,0,0,0 for labels,features in dataloader: optimizer.zero_grad() out = net(features) loss = loss_fn(out,labels) #cross_entropy(out,labels) loss.backward() optimizer.step() total_loss+=loss _,predicted = torch.max(out,1) acc+=(predicted==labels).sum() count+=len(labels) i+=1 if i%report_freq==0: print(f"{count}: acc={acc.item()/count}") if epoch_size and count>epoch_size: break return total_loss.item()/count, acc.item()/count

bigram_vectorizer = CountVectorizer(ngram_range=(1, 2), token_pattern=r'\b\w+\b', min_df=1) corpus = [ 'I like hot dogs.', 'The dog ran fast.', 'Its hot outside.', ] bigram_vectorizer.fit_transform(corpus) print("Vocabulary:\n",bigram_vectorizer.vocabulary_) bigram_vectorizer.transform(['My dog likes hot dogs on a hot day.']).toarray()

counter = collections.Counter() for label, line in train_dataset: counter.update(course_ngrams_iterator(tokenizer(line), ngrams=2)) bi_vocab = CourseVocabulary(counter, min_freq=1) print("Bigram vocabulary length = ", len(bi_vocab))

from sklearn.feature_extraction.text import TfidfVectorizer vectorizer = TfidfVectorizer(ngram_range=(1,2)) vectorizer.fit_transform(corpus) vectorizer.transform(['My dog likes hot dogs on a hot day.']).toarray()