$$heading_1$$ Numpy versus Pytorch¶

by Chuck Anderson, Pattern Exploration

Here the accuracy and computation time of the training of simple fully-connected neural networks using numpy and pytorch implementations and applied to the MNIST data set are compared. The Adam optimization algorithm in numpy and pytorch are compared, as well as the Scaled Conjugate Gradient optimization algorithm in numpy.

In :
!nvidia-smi


Mon Jul 17 15:16:36 2017
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 375.66                 Driver Version: 375.66                    |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  NVS 315             Off  | 0000:02:00.0     N/A |                  N/A |
| 30%   42C    P8    N/A /  N/A |     35MiB /   964MiB |     N/A      Default |
+-------------------------------+----------------------+----------------------+
|   1  GeForce GTX 980     Off  | 0000:03:00.0     Off |                  N/A |
| 26%   36C    P8    13W / 180W |    281MiB /  4038MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID  Type  Process name                               Usage      |
|=============================================================================|
|    0                  Not Supported                                         |
|    1      8001    C   .../anderson/anaconda3/envs/intel/bin/python   277MiB |
+-----------------------------------------------------------------------------+


In :
import numpy as np
import pickle
import gzip
import json
import time
import sys
import subprocess

# For my numpy neural network implementation
import neuralnetworks as nn

# for pytorch
import os
import torch
import torch.nn as tnn
import torchvision.datasets as dsets
import torchvision.transforms as transforms

# for reading and plotting results
import matplotlib.pyplot as plt
%matplotlib inline


In :
######################################################################
## Read mnist data into dataTrain

with gzip.open$$extract_itex$$'mnist.pkl.gz', 'rb'$$/extract_itex$$ as f:
train_set, valid_set, test_set = pickle.load$$extract_itex$$f, encoding='latin1'$$/extract_itex$$
Xtrain = np.array$$extract_itex$$train_set$$0$$$$/extract_itex$$
Xval = np.array$$extract_itex$$valid_set$$0$$$$/extract_itex$$
Xtest = np.array$$extract_itex$$test_set$$0$$$$/extract_itex$$
Ttrain = np.array$$extract_itex$$train_set$$1$$$$/extract_itex$$.reshape$$extract_itex$$(-1, 1$$/extract_itex$$)
Tval = np.array$$extract_itex$$valid_set$$1$$$$/extract_itex$$.reshape$$extract_itex$$(-1, 1$$/extract_itex$$)
Ttest = np.array$$extract_itex$$test_set$$1$$$$/extract_itex$$.reshape$$extract_itex$$(-1, 1$$/extract_itex$$)

# to match with main-gpu.py in res/pytorch...01../feed.../main-gpu.py
Xtrain = np.vstack$$extract_itex$$(Xtrain, Xval$$/extract_itex$$)
Ttrain = np.vstack$$extract_itex$$(Ttrain, Tval$$/extract_itex$$)
dataTrain = np.hstack$$extract_itex$$(Xtrain, Ttrain$$/extract_itex$$)  # so we can shuffle the order
dataTrain = dataTrain.astype$$extract_itex$$np.float32$$/extract_itex$$
nSamples = dataTrain.shape$$0$$


In :
######################################################################
## Write results

def writeResults$$extract_itex$$filename, timeAcc, label$$/extract_itex$$:
if filename == 'stdout':
f = sys.stdout
else:
f = open$$extract_itex$$filename, 'a'$$/extract_itex$$
f.write$$extract_itex$$label+'\n'$$/extract_itex$$
f.write$$extract_itex$$str(len(timeAcc$$/extract_itex$$) + '\n')
for ta in timeAcc:
f.write$$extract_itex$$'{:.2f} {:.3f}\n'.format(ta$$0$$, ta$$1$$$$/extract_itex$$)
if filename != 'stdout':
f.close


In :
def runnumpy$$extract_itex$$batchSize=None, numEpochs=10, hidden=$$100$$, nIterations=100, useRelu=False, useAdam=False$$/extract_itex$$:

label = 'Numpy '
label += ' batch {} epochs {:d} hids {} nIter {:d}'.format$$extract_itex$$batchSize, numEpochs, hidden, nIterations$$/extract_itex$$
label += ' ReLU ' if useRelu else ' Tanh '
label += time.strftime$$extract_itex$$'%m/%d/17-%H:%M'$$/extract_itex$$

Xtrain = dataTrain$$:,:-1$$
Ttrain = dataTrain$$:,-1:$$

nnet = nn.NeuralNetworkClassifier$$extract_itex$$$$Xtrain.shape[1$$] + hidden + $$10$$, np.arange(10$$/extract_itex$$, useRelu=useRelu)
# NOT STANDARDIZING THE INPUTS!!!
nnet.setStandardize$$extract_itex$$False$$/extract_itex$$

secsAcc = 

# numEpochs determines number of breaks during training to calculate test error
if batchSize is None:

startTime = time.time
for i in range$$extract_itex$$numEpochs$$/extract_itex$$:
Xtrain = dataTrain$$:, :-1$$
Ttrain = dataTrain$$:, -1:$$
nnet.train$$extract_itex$$Xtrain, Ttrain, nIterations=nIterations, verbose=False$$/extract_itex$$
ptest = nnet.use$$extract_itex$$Xtest$$/extract_itex$$
secsAcc.append$$extract_itex$$$$time.time([/extract_itex$$ - startTime, np.mean$$extract_itex$$ptest!=Ttest$$/extract_itex$$])

else:  # numpyg on batches

nSamples = dataTrain.shape$$0$$
if nSamples % batchSize != 0:
print$$extract_itex$$'WARNING: nSamples {} is not divisible by batchSize {}'.format( nSampmles, batchSize$$/extract_itex$$)
nBatches = nSamples // batchSize
startTime = time.time
for epoch in range$$extract_itex$$numEpochs$$/extract_itex$$:
np.random.shuffle$$extract_itex$$dataTrain$$/extract_itex$$
for traini in range$$extract_itex$$0, nSamples, batchSize$$/extract_itex$$:
Xtrain = dataTrain$$traini:traini+batchSize,:-1$$
Ttrain = dataTrain$$traini:traini+batchSize,-1:$$
nnet.train$$extract_itex$$Xtrain, Ttrain, restart=True, nIterations=nIterations, useAdam=useAdam$$/extract_itex$$
ptest = nnet.use$$extract_itex$$Xtest$$/extract_itex$$
secsAcc.append$$extract_itex$$$$time.time([/extract_itex$$ - startTime, np.mean$$extract_itex$$ptest!=Ttest$$/extract_itex$$])

writeResults$$extract_itex$$resultsFilename, secsAcc, label$$/extract_itex$$
if numEpochs <= 10:
writeResults$$extract_itex$$'stdout', secsAcc, label$$/extract_itex$$


In :
def runpytorch$$extract_itex$$batchSize=100, numEpochs=10, hidden=$$100$$, learningRate=0.001, nIterations=100, useRelu=True, useGPU=False$$/extract_itex$$:

if useGPU:
os.environ$$'CUDA_VISIBLE_DEVICES'$$ = '0'
print$$extract_itex$$'torch.cuda.is_available($$/extract_itex$$ is', torch.cuda.is_available)

if not torch.cuda.is_available:
print$$extract_itex$$'GPU is not available. Not running sgd pytorch cpu'$$/extract_itex$$
return

label = 'Pytorch '
if useGPU:
label += 'GPU '
label += 'Adam batch {} epochs {:d} lr {:.6f} hids {} nIter {:d}'.format$$extract_itex$$batchSize, numEpochs, learningRate, hidden, nIterations$$/extract_itex$$
label += ' ReLU ' if useRelu else ' Tanh '
label += time.strftime$$extract_itex$$'%m/%d/17-%H:%M'$$/extract_itex$$

# Neural Network Model $$extract_itex$$1 hidden layer$$/extract_itex$$
class Net$$extract_itex$$tnn.Module$$/extract_itex$$:
def __init__$$extract_itex$$self, input_size, hidden_size, num_classes$$/extract_itex$$:
self.hidden_size = hidden_size
super$$extract_itex$$Net, self$$/extract_itex$$.__init__
self.fc1 = tnn.Linear$$extract_itex$$input_size, hidden_size$$0$$$$/extract_itex$$
self.relu = tnn.ReLU if useRelu else tnn.Tanh
if len$$extract_itex$$hidden_size$$/extract_itex$$ > 1:
self.fc2 = tnn.Linear$$extract_itex$$hidden_size$$0$$, hidden_size$$1$$$$/extract_itex$$
self.relu2 = tnn.ReLU if useRelu else tnn.Tanh
self.fc3 = tnn.Linear$$extract_itex$$hidden_size$$1$$, num_classes$$/extract_itex$$
else:
self.fc3 = tnn.Linear$$extract_itex$$hidden_size$$0$$, num_classes$$/extract_itex$$
def forward$$extract_itex$$self, x$$/extract_itex$$:
out = self.fc1$$extract_itex$$x$$/extract_itex$$
out = self.relu$$extract_itex$$out$$/extract_itex$$
if len$$extract_itex$$self.hidden_size$$/extract_itex$$ > 1:
out = self.fc2$$extract_itex$$out$$/extract_itex$$
out = self.relu2$$extract_itex$$out$$/extract_itex$$
out = self.fc3$$extract_itex$$out$$/extract_itex$$
return out

train_dataset = dsets.MNIST$$extract_itex$$root='./data', train=True, transform=transforms.ToTensor($$/extract_itex$$,

test_dataset = dsets.MNIST$$extract_itex$$root='./data', train=False, transform=transforms.ToTensor($$/extract_itex$$)

# Data Loader $$extract_itex$$Input Pipeline$$/extract_itex$$
train_loader = torch.utils.data.DataLoader$$extract_itex$$dataset=train_dataset, batch_size=batchSize, shuffle=True$$/extract_itex$$

test_loader = torch.utils.data.DataLoader$$extract_itex$$dataset=test_dataset, batch_size=batchSize, shuffle=False$$/extract_itex$$

num_classes = 10
net = Net$$extract_itex$$784, hidden, num_classes$$/extract_itex$$
if useGPU:
net.cuda

# Loss and Optimizer
criterion = tnn.CrossEntropyLoss
optimizer = torch.optim.Adam$$extract_itex$$net.parameters($$/extract_itex$$, lr=learningRate)

global dataTrain, nSamples

if nSamples % batchSize != 0:
print$$extract_itex$$'WARNING: nSamples {} is not divisible by batchSize {}'.format( nSampmles, batchSize$$/extract_itex$$)
nBatches = nSamples // batchSize

secsAcc = 
startTime = time.time
for epoch in range$$extract_itex$$numEpochs$$/extract_itex$$:
np.random.shuffle$$extract_itex$$dataTrain$$/extract_itex$$
for i, $$extract_itex$$images, labels$$/extract_itex$$ in enumerate$$extract_itex$$train_loader$$/extract_itex$$:
# Forward + Backward + Optimize
if useGPU:
images = Variable$$extract_itex$$images.view(-1, 28*28$$/extract_itex$$).cuda
labels = Variable$$extract_itex$$labels$$/extract_itex$$.cuda
else:
images = Variable$$extract_itex$$images.view(-1, 28*28$$/extract_itex$$)
labels = Variable$$extract_itex$$labels$$/extract_itex$$
# Forward + Backward + Optimize
for iter in range$$extract_itex$$nIterations$$/extract_itex$$:
optimizer.zero_grad  # zero the gradient buffer
outputs = net$$extract_itex$$images$$/extract_itex$$
loss = criterion$$extract_itex$$outputs, labels$$/extract_itex$$
loss.backward
optimizer.step

correct = 0
total = 0
if useGPU:
images = Variable$$extract_itex$$images.view(-1, 28*28$$/extract_itex$$).cuda
else:
images = Variable$$extract_itex$$images.view(-1, 28*28$$/extract_itex$$)
outputs = net$$extract_itex$$images$$/extract_itex$$
_, predicted = torch.max$$extract_itex$$outputs.data, 1$$/extract_itex$$
total += labels.size$$extract_itex$$0$$/extract_itex$$
if useGPU:
correct += $$extract_itex$$predicted.cpu($$/extract_itex$$ == labels).sum
else:
correct += $$extract_itex$$predicted == labels$$/extract_itex$$.sum
secsAcc.append$$extract_itex$$$$time.time([/extract_itex$$ - startTime, $$extract_itex$$total-correct$$/extract_itex$$ / total])

writeResults$$extract_itex$$resultsFilename, secsAcc, label$$/extract_itex$$
if numEpochs <= 10:
writeResults$$extract_itex$$'stdout', secsAcc, label$$/extract_itex$$


In :
resultsFilename = 'test.results'
subprocess.call$$extract_itex$$$$'rm', resultsFilename$$$$/extract_itex$$
hidden = $$500, 500$$  # can contain one or two ints, for one or two hidden layers
batchSize = 100
numEpochs = 50


In :
runpytorch$$extract_itex$$batchSize=batchSize, numEpochs=numEpochs, hidden=hidden, learningRate=0.001, nIterations=1, useRelu=False, useGPU=True$$/extract_itex$$


torch.cuda.is_available is True


In :
runpytorch$$extract_itex$$batchSize=batchSize, numEpochs=numEpochs, hidden=hidden, learningRate=0.001, nIterations=1, useRelu=True, useGPU=False$$/extract_itex$$


In :
runpytorch$$extract_itex$$batchSize=batchSize, numEpochs=numEpochs, hidden=hidden, learningRate=0.001, nIterations=1, useRelu=False, useGPU=False$$/extract_itex$$


In :
runpytorch$$extract_itex$$batchSize=batchSize, numEpochs=numEpochs, hidden=hidden, learningRate=0.001, nIterations=1, useRelu=True, useGPU=False$$/extract_itex$$


In :
def plotFromFile$$extract_itex$$filename='test.results'$$/extract_itex$$:
results = {}
with open$$extract_itex$$filename,'r'$$/extract_itex$$ as f:
while True:
label = f.readline
if label is None or label == '':
break;
n = int$$extract_itex$$f.readline($$/extract_itex$$)
secsAcc = 
for i in range$$extract_itex$$n$$/extract_itex$$:
secsAcc.append$$extract_itex$$$$float(s[/extract_itex$$ for s in f.readline.split$$extract_itex$$' '$$/extract_itex$$])
results$$label$$ = secsAcc

markers = $$'s','8','>','^','<','v','o','X','P','d','h','*','p','D','H'$$
mi = 0
print$$extract_itex$$sorted(results$$/extract_itex$$)
for key in sorted$$extract_itex$$results$$/extract_itex$$:
value = results$$key$$
value = np.array$$extract_itex$$value$$/extract_itex$$
plt.plot$$extract_itex$$value$$:, 0$$, value$$:, 1$$, '-', marker=markers$$mi$$, label=key, lw=4, markersize=15$$/extract_itex$$
mi = $$extract_itex$$mi + 1$$/extract_itex$$ % len$$extract_itex$$markers$$/extract_itex$$
plt.xlabel$$extract_itex$$'Seconds'$$/extract_itex$$
plt.ylabel$$extract_itex$$'Fraction of test samples incorrectly classified'$$/extract_itex$$
plt.legend;


In :
# cat test.results


In :
plt.figure$$extract_itex$$figsize=(20, 12$$/extract_itex$$)
plotFromFile$$extract_itex$$'test.results'$$/extract_itex$$


$$'Pytorch Adam batch 100 epochs 50 lr 0.001000 hids [500, 500$$ nIter 1 ReLU 07/17/17-15:20\n', 'Pytorch Adam batch 100 epochs 50 lr 0.001000 hids $$500, 500$$ nIter 1 ReLU 07/17/17-16:03\n', 'Pytorch Adam batch 100 epochs 50 lr 0.001000 hids $$500, 500$$ nIter 1 Tanh 07/17/17-15:42\n', 'Pytorch GPU Adam batch 100 epochs 50 lr 0.001000 hids $$500, 500$$ nIter 1 Tanh 07/17/17-15:16\n']


In :
runnumpy$$extract_itex$$batchSize=batchSize, numEpochs=numEpochs, hidden=hidden, nIterations=1, useRelu=False, useAdam=True$$/extract_itex$$
runnumpy$$extract_itex$$batchSize=batchSize, numEpochs=numEpochs, hidden=hidden, nIterations=1, useRelu=False, useAdam=False$$/extract_itex$$

runnumpy$$extract_itex$$batchSize=batchSize, numEpochs=numEpochs, hidden=hidden, nIterations=1, useRelu=True, useAdam=True$$/extract_itex$$
runnumpy$$extract_itex$$batchSize=batchSize, numEpochs=numEpochs, hidden=hidden, nIterations=1, useRelu=True, useAdam=False$$/extract_itex$$


In :
runnumpy$$extract_itex$$batchSize=None, numEpochs=50, hidden=hidden, nIterations=10, useRelu=False, useAdam=True$$/extract_itex$$
runnumpy$$extract_itex$$batchSize=None, numEpochs=50, hidden=hidden, nIterations=10, useRelu=False, useAdam=False$$/extract_itex$$

runnumpy$$extract_itex$$batchSize=None, numEpochs=50, hidden=hidden, nIterations=10, useRelu=True, useAdam=True$$/extract_itex$$
runnumpy$$extract_itex$$batchSize=None, numEpochs=50, hidden=hidden, nIterations=10, useRelu=True, useAdam=False$$/extract_itex$$


In :
plt.figure$$extract_itex$$figsize=(20, 12$$/extract_itex$$)
plotFromFile$$extract_itex$$'test.results'$$/extract_itex$$


$$'Numpy Adam batch 100 epochs 50 hids [500, 500$$ nIter 1 ReLU 07/17/17-16:35\n', 'Numpy Adam batch 100 epochs 50 hids $$500, 500$$ nIter 1 Tanh 07/17/17-16:25\n', 'Numpy Adam batch None epochs 50 hids $$500, 500$$ nIter 10 ReLU 07/17/17-17:29\n', 'Numpy Adam batch None epochs 50 hids $$500, 500$$ nIter 10 Tanh 07/17/17-16:47\n', 'Numpy SCG batch 100 epochs 50 hids $$500, 500$$ nIter 1 ReLU 07/17/17-16:40\n', 'Numpy SCG batch 100 epochs 50 hids $$500, 500$$ nIter 1 Tanh 07/17/17-16:29\n', 'Numpy SCG batch None epochs 50 hids $$500, 500$$ nIter 10 ReLU 07/17/17-17:57\n', 'Numpy SCG batch None epochs 50 hids $$500, 500$$ nIter 10 Tanh 07/17/17-17:08\n', 'Pytorch Adam batch 100 epochs 50 lr 0.001000 hids $$500, 500$$ nIter 1 ReLU 07/17/17-15:20\n', 'Pytorch Adam batch 100 epochs 50 lr 0.001000 hids $$500, 500$$ nIter 1 ReLU 07/17/17-16:03\n', 'Pytorch Adam batch 100 epochs 50 lr 0.001000 hids $$500, 500$$ nIter 1 Tanh 07/17/17-15:42\n', 'Pytorch GPU Adam batch 100 epochs 50 lr 0.001000 hids $$500, 500$$ nIter 1 Tanh 07/17/17-15:16\n']


In :
plt.figure$$extract_itex$$figsize=(20, 12$$/extract_itex$$)
plotFromFile$$extract_itex$$'test.results'$$/extract_itex$$
plt.ylim$$extract_itex$$0.01,0.125$$/extract_itex$$
# plt.xlim$$extract_itex$$0,40$$/extract_itex$$


$$'Numpy Adam batch 100 epochs 50 hids [500, 500$$ nIter 1 ReLU 07/17/17-16:35\n', 'Numpy Adam batch 100 epochs 50 hids $$500, 500$$ nIter 1 Tanh 07/17/17-16:25\n', 'Numpy Adam batch None epochs 50 hids $$500, 500$$ nIter 10 ReLU 07/17/17-17:29\n', 'Numpy Adam batch None epochs 50 hids $$500, 500$$ nIter 10 Tanh 07/17/17-16:47\n', 'Numpy SCG batch 100 epochs 50 hids $$500, 500$$ nIter 1 ReLU 07/17/17-16:40\n', 'Numpy SCG batch 100 epochs 50 hids $$500, 500$$ nIter 1 Tanh 07/17/17-16:29\n', 'Numpy SCG batch None epochs 50 hids $$500, 500$$ nIter 10 ReLU 07/17/17-17:57\n', 'Numpy SCG batch None epochs 50 hids $$500, 500$$ nIter 10 Tanh 07/17/17-17:08\n', 'Pytorch Adam batch 100 epochs 50 lr 0.001000 hids $$500, 500$$ nIter 1 ReLU 07/17/17-15:20\n', 'Pytorch Adam batch 100 epochs 50 lr 0.001000 hids $$500, 500$$ nIter 1 ReLU 07/17/17-16:03\n', 'Pytorch Adam batch 100 epochs 50 lr 0.001000 hids $$500, 500$$ nIter 1 Tanh 07/17/17-15:42\n', 'Pytorch GPU Adam batch 100 epochs 50 lr 0.001000 hids $$500, 500$$ nIter 1 Tanh 07/17/17-15:16\n']


Out:
$$extract_itex$$0.01, 0.125$$/extract_itex$$

In :
plt.figure$$extract_itex$$figsize=(20, 12$$/extract_itex$$)
plotFromFile$$extract_itex$$'test.results'$$/extract_itex$$
plt.ylim$$extract_itex$$0.01,0.04$$/extract_itex$$
# plt.xlim$$extract_itex$$0,40$$/extract_itex$$


$$'Numpy Adam batch 100 epochs 50 hids [500, 500$$ nIter 1 ReLU 07/17/17-16:35\n', 'Numpy Adam batch 100 epochs 50 hids $$500, 500$$ nIter 1 Tanh 07/17/17-16:25\n', 'Numpy Adam batch None epochs 50 hids $$500, 500$$ nIter 10 ReLU 07/17/17-17:29\n', 'Numpy Adam batch None epochs 50 hids $$500, 500$$ nIter 10 Tanh 07/17/17-16:47\n', 'Numpy SCG batch 100 epochs 50 hids $$500, 500$$ nIter 1 ReLU 07/17/17-16:40\n', 'Numpy SCG batch 100 epochs 50 hids $$500, 500$$ nIter 1 Tanh 07/17/17-16:29\n', 'Numpy SCG batch None epochs 50 hids $$500, 500$$ nIter 10 ReLU 07/17/17-17:57\n', 'Numpy SCG batch None epochs 50 hids $$500, 500$$ nIter 10 Tanh 07/17/17-17:08\n', 'Pytorch Adam batch 100 epochs 50 lr 0.001000 hids $$500, 500$$ nIter 1 ReLU 07/17/17-15:20\n', 'Pytorch Adam batch 100 epochs 50 lr 0.001000 hids $$500, 500$$ nIter 1 ReLU 07/17/17-16:03\n', 'Pytorch Adam batch 100 epochs 50 lr 0.001000 hids $$500, 500$$ nIter 1 Tanh 07/17/17-15:42\n', 'Pytorch GPU Adam batch 100 epochs 50 lr 0.001000 hids $$500, 500$$ nIter 1 Tanh 07/17/17-15:16\n']


Out:
$$extract_itex$$0.01, 0.04$$/extract_itex$$