Perceptron

import pylab
from matplotlib import gridspec
from sklearn.datasets import load_digits, make_classification
from types import SimpleNamespace
import numpy as np
from ipywidgets import interact, interactive, fixed
import ipywidgets as widgets
# pick the seed for reproducability - change it to explore the effects of random variations
np.random.seed(1)
import random

n = 50
X, Y = make_classification(n_samples = n, n_features=2,
                           n_redundant=0, n_informative=2, flip_y=0)
Y = Y*2-1 # convert initial 0/1 values into -1/1
X = X.astype(np.float32); Y = Y.astype(np.int32) # features - float, label - int

# Split the dataset into training and test
train_x, test_x = np.split(X, [ n*8//10])
train_labels, test_labels = np.split(Y, [n*8//10])
print("Features:\n",train_x[0:4])
print("Labels:\n",train_labels[0:4])

def plot_dataset(suptitle, features, labels):
    # prepare the plot
    fig, ax = pylab.subplots(1, 1)
    #pylab.subplots_adjust(bottom=0.2, wspace=0.4)
    fig.suptitle(suptitle, fontsize = 16)
    ax.set_xlabel('$x_i[0]$ -- (feature 1)')
    ax.set_ylabel('$x_i[1]$ -- (feature 2)')

    colors = ['r' if l>0 else 'b' for l in labels]
    ax.scatter(features[:, 0], features[:, 1], marker='o', c=colors, s=100, alpha = 0.5)
    fig.show()

plot_dataset('Training data', train_x, train_labels)

pos_examples = np.array([ [t[0], t[1], 1] for i,t in enumerate(train_x)
                          if train_labels[i]>0])
neg_examples = np.array([ [t[0], t[1], 1] for i,t in enumerate(train_x)
                          if train_labels[i]<0])
print(pos_examples[0:3])

def train(positive_examples, negative_examples, num_iterations = 100, learning_rate = 0.01):
    num_dims = positive_examples.shape[1]

    # Initialize weights.
    # We initialize with 0 for simplicity, but random initialization is also a good idea
    weights = np.zeros((num_dims,1))

    pos_count = positive_examples.shape[0]
    neg_count = negative_examples.shape[0]

    report_frequency = 10

    for i in range(num_iterations):
        # Pick one positive and one negative example
        pos = random.choice(positive_examples)
        neg = random.choice(negative_examples)

        z = np.dot(pos, weights)
        if z < 0: # positive example was classified as negative
            weights = weights + learning_rate * pos.reshape(weights.shape)

        z  = np.dot(neg, weights)
        if z >= 0: # negative example was classified as positive
            weights = weights - learning_rate * neg.reshape(weights.shape)

        # Periodically, print out the current accuracy on all examples
        if i % report_frequency == 0:
            pos_out = np.dot(positive_examples, weights)
            neg_out = np.dot(negative_examples, weights)
            pos_correct = (pos_out >= 0).sum() / float(pos_count)
            neg_correct = (neg_out < 0).sum() / float(neg_count)
            print("Iteration={}, pos correct={}, neg correct={}".format(i,pos_correct,neg_correct))

    return weights

wts = train(pos_examples,neg_examples)
print(wts.transpose())

def plot_boundary(positive_examples, negative_examples, weights):
    if np.isclose(weights[1], 0):
        if np.isclose(weights[0], 0):
            x = y = np.array([-6, 6], dtype = 'float32')
        else:
            y = np.array([-6, 6], dtype='float32')
            x = -(weights[1] * y + weights[2])/weights[0]
    else:
        x = np.array([-6, 6], dtype='float32')
        y = -(weights[0] * x + weights[2])/weights[1]

    pylab.xlim(-6, 6)
    pylab.ylim(-6, 6)
    pylab.plot(positive_examples[:,0], positive_examples[:,1], 'bo')
    pylab.plot(negative_examples[:,0], negative_examples[:,1], 'ro')
    pylab.plot(x, y, 'g', linewidth=2.0)
    pylab.show()

plot_boundary(pos_examples,neg_examples,wts)

# Compare different learning rates
learning_rates = [0.001, 0.01, 0.1, 1.0]
fig, axes = pylab.subplots(2, 2, figsize=(12, 10))
fig.suptitle('Effect of Different Learning Rates', fontsize=16)

for idx, lr in enumerate(learning_rates):
    ax = axes[idx // 2, idx % 2]

    # Train with this learning rate
    weights_lr = train(pos_examples, neg_examples, num_iterations=100, learning_rate=lr)

    # Plot decision boundary
    if np.isclose(weights_lr[1], 0):
        if np.isclose(weights_lr[0], 0):
            x = y = np.array([-6, 6], dtype='float32')
        else:
            y = np.array([-6, 6], dtype='float32')
            x = -(weights_lr[1] * y + weights_lr[2])/weights_lr[0]
    else:
        x = np.array([-6, 6], dtype='float32')
        y = -(weights_lr[0] * x + weights_lr[2])/weights_lr[1]

    ax.set_xlim(-6, 6)
    ax.set_ylim(-6, 6)
    ax.plot(pos_examples[:, 0], pos_examples[:, 1], 'bo', label='Positive', alpha=0.7)
    ax.plot(neg_examples[:, 0], neg_examples[:, 1], 'ro', label='Negative', alpha=0.7)
    ax.plot(x, y, 'g-', linewidth=2)
    ax.set_title(f'Learning Rate = {lr}')
    ax.set_xlabel('Feature 1')
    ax.set_ylabel('Feature 2')
    ax.legend()
    ax.grid(True, alpha=0.3)

pylab.tight_layout()
pylab.show()

def train_and_plot_with_lr(learning_rate=0.01):
    """Train perceptron with specified learning rate and plot results"""
    weights_lr = train(pos_examples, neg_examples, num_iterations=100, learning_rate=learning_rate)

    fig, (ax1, ax2) = pylab.subplots(1, 2, figsize=(14, 5))

    # Plot 1: Decision boundary
    if np.isclose(weights_lr[1], 0):
        if np.isclose(weights_lr[0], 0):
            x = y = np.array([-6, 6], dtype='float32')
        else:
            y = np.array([-6, 6], dtype='float32')
            x = -(weights_lr[1] * y + weights_lr[2])/weights_lr[0]
    else:
        x = np.array([-6, 6], dtype='float32')
        y = -(weights_lr[0] * x + weights_lr[2])/weights_lr[1]

    ax1.set_xlim(-6, 6)
    ax1.set_ylim(-6, 6)
    ax1.plot(pos_examples[:, 0], pos_examples[:, 1], 'bo', label='Positive', s=100, alpha=0.6)
    ax1.plot(neg_examples[:, 0], neg_examples[:, 1], 'ro', label='Negative', s=100, alpha=0.6)
    ax1.plot(x, y, 'g-', linewidth=3, label='Decision Boundary')
    ax1.set_title(f'Decision Boundary (lr={learning_rate})', fontsize=14)
    ax1.set_xlabel('Feature 1')
    ax1.set_ylabel('Feature 2')
    ax1.legend()
    ax1.grid(True, alpha=0.3)

    # Plot 2: Weight values
    ax2.bar(['w0', 'w1', 'bias'], weights_lr.flatten(), color=['blue', 'green', 'red'], alpha=0.7)
    ax2.set_title('Final Weight Values', fontsize=14)
    ax2.set_ylabel('Weight Value')
    ax2.grid(True, alpha=0.3, axis='y')
    ax2.axhline(y=0, color='black', linestyle='-', linewidth=0.5)

    pylab.tight_layout()
    pylab.show()

    print(f"Final weights: {weights_lr.flatten()}")

# Create interactive widget
interact(train_and_plot_with_lr,
         learning_rate=widgets.FloatSlider(value=0.01, min=0.001, max=1.0, step=0.001,
                                          description='Learning Rate:', continuous_update=False))

def accuracy(weights, test_x, test_labels):
    res = np.dot(np.c_[test_x,np.ones(len(test_x))],weights)
    return (res.reshape(test_labels.shape)*test_labels>=0).sum()/float(len(test_labels))

accuracy(wts, test_x, test_labels)

def train_graph(positive_examples, negative_examples, num_iterations = 100, learning_rate = 0.01):
    num_dims = positive_examples.shape[1]
    weights = np.zeros((num_dims,1)) # initialize weights

    pos_count = positive_examples.shape[0]
    neg_count = negative_examples.shape[0]

    report_frequency = 15;
    snapshots = []

    for i in range(num_iterations):
        pos = random.choice(positive_examples)
        neg = random.choice(negative_examples)

        z = np.dot(pos, weights)
        if z < 0:
            weights = weights + learning_rate * pos.reshape(weights.shape)

        z  = np.dot(neg, weights)
        if z >= 0:
            weights = weights - learning_rate * neg.reshape(weights.shape)

        if i % report_frequency == 0:
            pos_out = np.dot(positive_examples, weights)
            neg_out = np.dot(negative_examples, weights)
            pos_correct = (pos_out >= 0).sum() / float(pos_count)
            neg_correct = (neg_out < 0).sum() / float(neg_count)
            snapshots.append([np.copy(weights).flatten(), (pos_correct+neg_correct)/2.0])

    return np.array(snapshots, dtype=object)

snapshots = train_graph(pos_examples,neg_examples)

def plotit(pos_examples,neg_examples,snapshots,step):
    fig = pylab.figure(figsize=(10,4))
    fig.add_subplot(1, 2, 1)
    plot_boundary(pos_examples, neg_examples, snapshots[step][0])
    fig.add_subplot(1, 2, 2)
    pylab.plot(np.arange(len(snapshots[:,1])), snapshots[:,1])
    pylab.ylabel('Accuracy')
    pylab.xlabel('Iteration')
    pylab.plot(step, snapshots[step,1], "bo")
    pylab.show()
def pl1(step): plotit(pos_examples,neg_examples,snapshots,step)

interact(pl1, step=widgets.IntSlider(value=0, min=0, max=len(snapshots)-1))

pos_examples_xor = np.array([[1,0,1],[0,1,1]])
neg_examples_xor = np.array([[1,1,1],[0,0,1]])

snapshots_xor = train_graph(pos_examples_xor,neg_examples_xor,1000)
def pl2(step): plotit(pos_examples_xor,neg_examples_xor,snapshots_xor,step)

interact(pl2, step=widgets.IntSlider(value=0, min=0, max=len(snapshots)-1))

_course_digits = load_digits()
_course_images = np.kron(_course_digits.images.astype(np.float32) / 16.0, np.ones((3, 3), dtype=np.float32))
_course_images = np.pad(_course_images, ((0, 0), (2, 2), (2, 2)))
mnist = SimpleNamespace(
    data=(_course_images.reshape(len(_course_images), -1) * 255.0).astype(np.float32),
    target=np.asarray(_course_digits.target, dtype=np.int64),
)
all_features = np.asarray(mnist.data, dtype=np.float32)
all_labels = np.asarray(mnist.target, dtype=np.int64)
train_size = int(0.8 * len(all_features))
MNIST = {
    "Train": {
        "Features": all_features[:train_size],
        "Labels": all_labels[:train_size],
    },
    "Test": {
        "Features": all_features[train_size:],
        "Labels": all_labels[train_size:],
    },
}

print(MNIST['Train']['Features'][0][130:180])
print(MNIST['Train']['Labels'][0])
features = MNIST['Train']['Features'].astype(np.float32) / 255.0
labels = MNIST['Train']['Labels']
fig = pylab.figure(figsize=(10,5))
for i in range(10):
    ax = fig.add_subplot(1,10,i+1)
    pylab.imshow(features[i].reshape(28,28))
pylab.show()

def set_mnist_pos_neg(positive_label, negative_label):
    positive_indices = [i for i, j in enumerate(MNIST['Train']['Labels'])
                          if j == positive_label]
    negative_indices = [i for i, j in enumerate(MNIST['Train']['Labels'])
                          if j == negative_label]

    positive_images = MNIST['Train']['Features'][positive_indices]
    negative_images = MNIST['Train']['Features'][negative_indices]

    fig = pylab.figure()
    ax = fig.add_subplot(1, 2, 1)
    pylab.imshow(positive_images[0].reshape(28,28), cmap='gray', interpolation='nearest')
    ax.set_xticks([])
    ax.set_yticks([])
    ax = fig.add_subplot(1, 2, 2)
    pylab.imshow(negative_images[0].reshape(28,28), cmap='gray', interpolation='nearest')
    ax.set_xticks([])
    ax.set_yticks([])
    pylab.show()

    return positive_images, negative_images

pos1,neg1 = set_mnist_pos_neg(1,0)

def plotit2(snapshots_mn,step):
    fig = pylab.figure(figsize=(10,4))
    ax = fig.add_subplot(1, 2, 1)
    pylab.imshow(snapshots_mn[step][0].reshape(28, 28), interpolation='nearest')
    ax.set_xticks([])
    ax.set_yticks([])
    pylab.colorbar()
    ax = fig.add_subplot(1, 2, 2)
    ax.set_ylim([0,1])
    pylab.plot(np.arange(len(snapshots_mn[:,1])), snapshots_mn[:,1])
    pylab.plot(step, snapshots_mn[step,1], "bo")
    pylab.show()
def pl3(step): plotit2(snapshots_mn,step)
def pl4(step): plotit2(snapshots_mn2,step)

snapshots_mn = train_graph(pos1,neg1,1000)
interact(pl3, step=widgets.IntSlider(value=0, min=0, max=len(snapshots_mn) - 1))

pos2,neg2 = set_mnist_pos_neg(2,5)

snapshots_mn2 = train_graph(pos2,neg2,1000)
interact(pl4, step=widgets.IntSlider(value=0, min=0, max=len(snapshots_mn2) - 1))

from sklearn.decomposition import PCA

def pca_analysis(positive_label, negative_label):
    positive_images, negative_images = set_mnist_pos_neg(positive_label, negative_label)
    M = np.append(positive_images, negative_images, 0)

    mypca = PCA(n_components=2)
    mypca.fit(M)

    pos_points = mypca.transform(positive_images[:200])
    neg_points = mypca.transform(negative_images[:200])

    pylab.plot(pos_points[:,0], pos_points[:,1], 'bo')
    pylab.plot(neg_points[:,0], neg_points[:,1], 'ro')

pca_analysis(1,0)

pca_analysis(2,5)