run.py

import optimizers
import networks
from keras.datasets import mnist
import torch

if __name__ == "__main__":
    N_data = 1000
    N_iters = 1000
    (X_train, y_train), (X_test, y_test) = mnist.load_data()
    X_train, y_train = torch.tensor(X_train[:N_data,:,:], dtype=torch.float), torch.tensor(y_train[:N_data])
    def loss_func(data):
        loss_func = torch.nn.CrossEntropyLoss(reduction='none')
        relu = torch.nn.ReLU()
        return relu(loss_func(data[0], data[1])+torch.log(torch.tensor([0.5])))
    
    print("Adam with summation\n--------")
    sum_net = networks.SimpleNN() 
    adam_s = optimizers.adam_sum(sum_net, loss_func, 0.01)
    comp_set_card = adam_s.train((X_train, y_train),N_iters)
    X_test, y_test = torch.tensor(X_test, dtype=torch.float), torch.tensor(y_test)
    acc = networks.eval(sum_net, (X_test, y_test)) 
    print("Accuracy: {:.2f}%".format(acc*100))
    print("Compression set size: {}/{}".format(comp_set_card, len(y_train)))
    print("-------")
    
    print("Adam with subgradient\n--------")
    max_net = networks.SimpleNN() 
    adam_m = optimizers.adam_max(max_net, loss_func, 0.01)
    comp_set_card = adam_m.train((X_train, y_train),N_iters)
    X_test, y_test = torch.tensor(X_test, dtype=torch.float), torch.tensor(y_test)
    acc = networks.eval(max_net, (X_test, y_test)) 
    print("Accuracy: {:.2f}%".format(acc*100))
    print("Compression set size: {}/{}".format(comp_set_card, len(y_train)))
    print("-------")
    
    print("Subsurface\n--------")
    sub_net = networks.SimpleNN() 
    sub = optimizers.subsurface(sub_net, loss_func, 0.01)
    comp_set_card = sub.train((X_train, y_train),N_iters)
    X_test, y_test = torch.tensor(X_test, dtype=torch.float), torch.tensor(y_test)
    acc = networks.eval(sub_net, (X_test, y_test)) 
    print("Accuracy: {:.2f}%".format(acc*100))
    print("Compression set size: {}/{}".format(comp_set_card, len(y_train)))
    print("-------")