Text Representation TF

import tensorflow as tf
from tensorflow import keras

# In this tutorial, we will be training a lot of models. In order to use GPU memory cautiously,
# we will set tensorflow option to grow GPU memory allocation when required.
physical_devices = tf.config.list_physical_devices("GPU")
if len(physical_devices) > 0:
    tf.config.experimental.set_memory_growth(physical_devices[0], True)

# course-edition deterministic TensorFlow news fixture v1
_course_titles = tf.constant([
    "Regional leaders agree on a public health plan",
    "The home team wins the basketball final",
    "Markets rise after a strong business report",
    "Researchers release an efficient intelligence model",
    "Diplomats begin a new round of peace talks",
    "The tennis champion returns after an injury",
    "Retail sales improve with consumer confidence",
    "A software update improves privacy",
])
_course_descriptions = tf.constant([
    "World policy and international cooperation",
    "Sports results and player development",
    "Business revenue and economic indicators",
    "Science and technology research",
    "World diplomacy and humanitarian aid",
    "Sports training and competition",
    "Business investment and employment",
    "Science security and technology",
])
_course_labels = tf.constant([0, 1, 2, 3, 0, 1, 2, 3], dtype=tf.int64)
_course_records = {"title": _course_titles, "description": _course_descriptions, "label": _course_labels}
ds_train = tf.data.Dataset.from_tensor_slices(_course_records).take(6)
ds_test = tf.data.Dataset.from_tensor_slices(_course_records).skip(6)
dataset = {"train": ds_train, "test": ds_test}

ds_train = dataset['train']
ds_test = dataset['test']

print(f"Length of train dataset = {len(ds_train)}")
print(f"Length of test dataset = {len(ds_test)}")

classes = ['World', 'Sports', 'Business', 'Sci/Tech']

for i,x in zip(range(5),ds_train):
    print(f"{x['label']} ({classes[x['label']]}) -> {x['title']} {x['description']}")

vocab_size = 50000
vectorizer = keras.layers.TextVectorization(
    max_tokens=vocab_size,
    output_mode="int",
)
vectorizer.adapt(
    ds_train.take(500)
    .map(lambda item: item["title"] + " " + item["description"])
    .batch(256)
)

vocab = vectorizer.get_vocabulary()
vocab_size = len(vocab)
print(vocab[:10])
print(f"Length of vocabulary: {vocab_size}")

vectorizer('I love to play with my words')

from sklearn.feature_extraction.text import CountVectorizer
sc_vectorizer = CountVectorizer()
corpus = [
        'I like hot dogs.',
        'The dog ran fast.',
        'Its hot outside.',
    ]
sc_vectorizer.fit_transform(corpus)
sc_vectorizer.transform(['My dog likes hot dogs on a hot day.']).toarray()

def to_bow(text):
    return tf.reduce_sum(tf.one_hot(vectorizer(text),vocab_size),axis=0)

to_bow('My dog likes hot dogs on a hot day.').numpy()

batch_size = 128

ds_train_bow = ds_train.map(lambda x: (to_bow(x['title']+x['description']),x['label'])).batch(batch_size)
ds_test_bow = ds_test.map(lambda x: (to_bow(x['title']+x['description']),x['label'])).batch(batch_size)

model = keras.models.Sequential([
    keras.layers.Dense(4,activation='softmax',input_shape=(vocab_size,))
])
model.compile(loss='sparse_categorical_crossentropy',optimizer='adam',metrics=['acc'])
model.fit(ds_train_bow,validation_data=ds_test_bow)

def extract_text(x):
    return x['title']+' '+x['description']

def tupelize(x):
    return (extract_text(x),x['label'])

inp = keras.Input(shape=(1,),dtype=tf.string)
x = vectorizer(inp)
x = keras.ops.sum(keras.ops.one_hot(x, vocab_size), axis=1)
out = keras.layers.Dense(4,activation='softmax')(x)
model = keras.models.Model(inp,out)
model.summary()

model.compile(loss='sparse_categorical_crossentropy',optimizer='adam',metrics=['acc'])
model.fit(ds_train.map(tupelize).batch(batch_size),validation_data=ds_test.map(tupelize).batch(batch_size))

bigram_vectorizer = CountVectorizer(ngram_range=(1, 2), token_pattern=r'\b\w+\b', min_df=1)
corpus = [
        'I like hot dogs.',
        'The dog ran fast.',
        'Its hot outside.',
    ]
bigram_vectorizer.fit_transform(corpus)
print("Vocabulary:\n",bigram_vectorizer.vocabulary_)
bigram_vectorizer.transform(['My dog likes hot dogs on a hot day.']).toarray()

count_vectorizer = keras.layers.TextVectorization(
    max_tokens=vocab_size,
    output_mode="count",
    pad_to_max_tokens=True,
)
count_vectorizer.adapt(ds_train.take(500).map(extract_text).batch(256))

model = keras.Sequential([
    keras.Input(shape=(), dtype=tf.string),
    count_vectorizer,
    keras.layers.Dense(4, activation="softmax"),
])
model.compile(
    loss="sparse_categorical_crossentropy",
    optimizer="adam",
    metrics=["acc"],
)
model.fit(
    ds_train.map(tupelize).batch(batch_size),
    validation_data=ds_test.map(tupelize).batch(batch_size),
)

from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer(ngram_range=(1,2))
vectorizer.fit_transform(corpus)
vectorizer.transform(['My dog likes hot dogs on a hot day.']).toarray()

tfidf_vectorizer = keras.layers.TextVectorization(
    max_tokens=vocab_size,
    output_mode="tf_idf",
    pad_to_max_tokens=True,
)
tfidf_vectorizer.adapt(ds_train.take(500).map(extract_text).batch(256))

model = keras.Sequential([
    keras.Input(shape=(), dtype=tf.string),
    tfidf_vectorizer,
    keras.layers.Dense(4, activation="softmax"),
])
model.compile(
    loss="sparse_categorical_crossentropy",
    optimizer="adam",
    metrics=["acc"],
)
model.fit(
    ds_train.map(tupelize).batch(batch_size),
    validation_data=ds_test.map(tupelize).batch(batch_size),
)