NER-TF

import pandas as pd
from tensorflow import keras
import numpy as np

# Course-authored miniature named-entity dataset.
course_sentences = [
    (("Aziz", "B-per"), ("visited", "O"), ("Riyadh", "B-geo"), ("today", "O")),
    (("Sara", "B-per"), ("joined", "O"), ("Hala", "B-org"), ("Academy", "I-org")),
    (("Omar", "B-per"), ("spoke", "O"), ("in", "O"), ("Jeddah", "B-geo")),
    (("Noura", "B-per"), ("studies", "O"), ("artificial", "B-tim"), ("intelligence", "I-tim")),
    (("Microsoft", "B-org"), ("opened", "O"), ("a", "O"), ("lab", "O")),
    (("Layla", "B-per"), ("traveled", "O"), ("to", "O"), ("Diriyah", "B-geo")),
    (("Fahad", "B-per"), ("built", "O"), ("a", "O"), ("robot", "O")),
    (("Maha", "B-per"), ("attended", "O"), ("the", "O"), ("conference", "O")),
    (("Aramco", "B-org"), ("announced", "O"), ("a", "O"), ("project", "O")),
    (("Khalid", "B-per"), ("works", "O"), ("in", "O"), ("Dammam", "B-geo")),
    (("Reem", "B-per"), ("teaches", "O"), ("machine", "O"), ("learning", "O")),
    (("Saud", "B-per"), ("presented", "O"), ("at", "O"), ("KAUST", "B-org")),
]
records = []
for sentence_index, sentence in enumerate(course_sentences, start=1):
    for word_index, (word, tag) in enumerate(sentence):
        records.append({
            "Sentence #": f"Sentence: {sentence_index}" if word_index == 0 else np.nan,
            "Word": word,
            "Tag": tag,
        })
df = pd.DataFrame.from_records(records)
df.head()

PAD_TAG = "<PAD>"
tags = sorted(df["Tag"].dropna().unique().tolist())
tags[:10]

id2tag = {0: PAD_TAG, **{index + 1: tag for index, tag in enumerate(tags)}}
tag2id = {tag: index for index, tag in id2tag.items()}
PAD_TAG_ID = tag2id[PAD_TAG]

id2tag[PAD_TAG_ID]

PAD_WORD = "<PAD>"
OOV_WORD = "<OOV>"
words = sorted({str(word).lower() for word in df["Word"].dropna()})
id2word = {0: PAD_WORD, 1: OOV_WORD, **{index + 2: word for index, word in enumerate(words)}}
word2id = {word: index for index, word in id2word.items()}
PAD_WORD_ID = word2id[PAD_WORD]
OOV_WORD_ID = word2id[OOV_WORD]

X,Y = [],[]
s,t = [],[]
for i,row in df[['Sentence #','Word','Tag']].iterrows():
    if pd.isna(row['Sentence #']):
        s.append(row['Word'])
        t.append(row['Tag'])
    else:
        if len(s)>0:
            X.append(s)
            Y.append(t)
        s,t = [row['Word']],[row['Tag']]
X.append(s)
Y.append(t)

def vectorize(sequence):
    return [word2id.get(str(word).lower(), OOV_WORD_ID) for word in sequence]

def tagify(sequence):
    return [tag2id[tag] for tag in sequence]

Xv = list(map(vectorize, X))
Yv = list(map(tagify, Y))

Xv[0], Yv[0]

X_data = keras.utils.pad_sequences(Xv, padding="post", value=PAD_WORD_ID)
Y_data = keras.utils.pad_sequences(Yv, padding="post", value=PAD_TAG_ID)
token_weights = (Y_data != PAD_TAG_ID).astype("float32")

maxlen = X_data.shape[1]
vocab_size = len(word2id)
num_tags = len(tag2id)
model = keras.models.Sequential([
    keras.Input(shape=(maxlen,), dtype="int32"),
    keras.layers.Embedding(vocab_size, 24, mask_zero=True),
    keras.layers.Bidirectional(keras.layers.LSTM(24, return_sequences=True)),
    keras.layers.Bidirectional(keras.layers.LSTM(24, return_sequences=True)),
    keras.layers.Dense(num_tags, activation="softmax"),
])
model.compile(loss="sparse_categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
model.summary()

model.fit(X_data, Y_data, sample_weight=token_weights, epochs=2, validation_split=0.1)

sent = "John Smith went to Paris to attend a conference in cancer development institute"
words = sent.lower().split()
encoded = [word2id.get(word, OOV_WORD_ID) for word in words]
v = keras.utils.pad_sequences([encoded], padding="post", maxlen=maxlen, value=PAD_WORD_ID)
res = model(v, training=False)[0]

predicted = np.argmax(res.numpy(), axis=1)[:len(words)]
for tag_id, word in zip(predicted, words):
    print(f"{word} -> {id2tag.get(int(tag_id), '<UNKNOWN_TAG>')}")