"""WRITEME"""

def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2,2)):

"""

assert image_shape[1]==filter_shape[1]

self.input = input

# initialize weight values: the fan-in of each hidden neuron is

# restricted by the size of the receptive fields.

fan_in = numpy.prod(filter_shape[1:])

W_values = numpy.asarray( rng.uniform( \

low = -numpy.sqrt(3./fan_in), \

high = numpy.sqrt(3./fan_in), \

size = filter_shape), dtype = theano.config.floatX)

self.W = theano.shared(value = W_values)

# the bias is a 1D tensor -- one bias per output feature map

b_values = numpy.zeros((filter_shape[0],), dtype= theano.config.floatX)

self.b = theano.shared(value= b_values)

# convolve input feature maps with filters

conv_out = conv.conv2d(input, self.W,

filter_shape=filter_shape, image_shape=image_shape)

# downsample each feature map individually, using maxpooling

pooled_out = downsample.max_pool2D(conv_out, poolsize, ignore_border=True)

# add the bias term. Since the bias is a vector (1D array), we first

# reshape it to a tensor of shape (1,n_filters,1,1). Each bias will thus

# be broadcasted across mini-batches and feature map width & height

self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))

# store parameters of this layer

def __init__(self, rng, input, n_in, n_out):

"""

self.input = input

W_values = numpy.asarray( rng.uniform( \

low = -numpy.sqrt(6./(n_in+n_out)), \

high = numpy.sqrt(6./(n_in+n_out)), \

size = (n_in, n_out)), dtype = theano.config.floatX)

self.W = theano.shared(value = W_values)

b_values = numpy.zeros((n_out,), dtype= theano.config.floatX)

self.b = theano.shared(value= b_values)

self.output = T.tanh(T.dot(input, self.W) + self.b)

"""Multi-class Logistic Regression Class

def __init__(self, input, n_in, n_out):

""" Initialize the parameters of the logistic regression

"""

# initialize with 0 the weights W as a matrix of shape (n_in, n_out)

self.W = theano.shared( value=numpy.zeros((n_in,n_out),

dtype = theano.config.floatX) )

# initialize the baises b as a vector of n_out 0s

self.b = theano.shared( value=numpy.zeros((n_out,),

dtype = theano.config.floatX) )

# compute vector of class-membership probabilities in symbolic form

self.p_y_given_x = T.nnet.softmax(T.dot(input, self.W)+self.b)

# compute prediction as class whose probability is maximal in

# symbolic form

self.y_pred=T.argmax(self.p_y_given_x, axis=1)

# list of parameters for this layer

self.params = [self.W, self.b]

def negative_log_likelihood(self, y):

"""Return the mean of the negative log-likelihood of the prediction

return -T.mean(T.log(self.p_y_given_x)[T.arange(y.shape[0]),y])

def errors(self, y):

"""Return a float representing the number of errors in the minibatch

# check if y has same dimension of y_pred

if y.ndim != self.y_pred.ndim:

raise TypeError('y should have the same shape as self.y_pred',

('y', target.type, 'y_pred', self.y_pred.type))

# check if y is of the correct datatype

if y.dtype.startswith('int'):

# the T.neq operator returns a vector of 0s and 1s, where 1

# represents a mistake in prediction

return T.mean(T.neq(self.y_pred, y))

else:

# Load the dataset

f = gzip.open(fname,'rb')

train_set, valid_set, test_set = cPickle.load(f)

f.close()

# make minibatches of size 20

batch_size = 20 # sized of the minibatch

# Dealing with the training set

# get the list of training images (x) and their labels (y)

(train_set_x, train_set_y) = train_set

# initialize the list of training minibatches with empty list

train_batches = []

for i in xrange(0, len(train_set_x), batch_size):

# add to the list of minibatches the minibatch starting at

# position i, ending at position i+batch_size

# a minibatch is a pair ; the first element of the pair is a list

# of datapoints, the second element is the list of corresponding

# labels

train_batches = train_batches + \

[(train_set_x[i:i+batch_size], train_set_y[i:i+batch_size])]

# Dealing with the validation set

(valid_set_x, valid_set_y) = valid_set

# initialize the list of validation minibatches

valid_batches = []

for i in xrange(0, len(valid_set_x), batch_size):

valid_batches = valid_batches + \

[(valid_set_x[i:i+batch_size], valid_set_y[i:i+batch_size])]

# Dealing with the testing set

(test_set_x, test_set_y) = test_set

# initialize the list of testing minibatches

test_batches = []

for i in xrange(0, len(test_set_x), batch_size):

test_batches = test_batches + \

[(test_set_x[i:i+batch_size], test_set_y[i:i+batch_size])]

rng = numpy.random.RandomState(23455)

train_batches, valid_batches, test_batches = load_dataset(dataset)

ishape = (28,28) # this is the size of MNIST images

batch_size = 20 # sized of the minibatch

# allocate symbolic variables for the data

x = T.matrix('x') # rasterized images

y = T.lvector() # the labels are presented as 1D vector of [long int] labels

######################

# BUILD ACTUAL MODEL #

######################

# Reshape matrix of rasterized images of shape (batch_size,28*28)

# to a 4D tensor, compatible with our LeNetConvPoolLayer

layer0_input = x.reshape((batch_size,1,28,28))

# Construct the first convolutional pooling layer:

# filtering reduces the image size to (28-5+1,28-5+1)=(24,24)

# maxpooling reduces this further to (24/2,24/2) = (12,12)

# 4D output tensor is thus of shape (20,20,12,12)

layer0 = LeNetConvPoolLayer(rng, input=layer0_input,

image_shape=(batch_size,1,28,28),

filter_shape=(20,1,5,5), poolsize=(2,2))

# Construct the second convolutional pooling layer

# filtering reduces the image size to (12-5+1,12-5+1)=(8,8)

# maxpooling reduces this further to (8/2,8/2) = (4,4)

# 4D output tensor is thus of shape (20,50,4,4)

layer1 = LeNetConvPoolLayer(rng, input=layer0.output,

image_shape=(batch_size,20,12,12),

filter_shape=(50,20,5,5), poolsize=(2,2))

# the SigmoidalLayer being fully-connected, it operates on 2D matrices of

# shape (batch_size,num_pixels) (i.e matrix of rasterized images).

# This will generate a matrix of shape (20,32*4*4) = (20,512)

layer2_input = layer1.output.flatten(2)

# construct a fully-connected sigmoidal layer

layer2 = SigmoidalLayer(rng, input=layer2_input,

n_in=50*4*4, n_out=500)

# classify the values of the fully-connected sigmoidal layer

layer3 = LogisticRegression(input=layer2.output, n_in=500, n_out=10)

# the cost we minimize during training is the NLL of the model

cost = layer3.negative_log_likelihood(y)

# create a function to compute the mistakes that are made by the model

test_model = theano.function([x,y], layer3.errors(y))

# create a list of all model parameters to be fit by gradient descent

params = layer3.params+ layer2.params+ layer1.params + layer0.params

# create a list of gradients for all model parameters

grads = T.grad(cost, params)

# train_model is a function that updates the model parameters by SGD

# Since this model has many parameters, it would be tedious to manually

# create an update rule for each model parameter. We thus create the updates

# dictionary by automatically looping over all (params[i],grads[i]) pairs.

updates = {}

for param_i, grad_i in zip(params, grads):

updates[param_i] = param_i - learning_rate * grad_i

train_model = theano.function([x, y], cost, updates=updates)

###############

# TRAIN MODEL #

###############

n_minibatches = len(train_batches)

# early-stopping parameters

patience = 10000 # look as this many examples regardless

patience_increase = 2 # wait this much longer when a new best is

# found

improvement_threshold = 0.995 # a relative improvement of this much is

# considered significant

validation_frequency = n_minibatches # go through this many

# minibatche before checking the network

# on the validation set; in this case we

# check every epoch

best_params = None

best_validation_loss = float('inf')

best_iter = 0

test_score = 0.

start_time = time.clock()

# have a maximum of `n_iter` iterations through the entire dataset

for iter in xrange(n_iter * n_minibatches):

# get epoch and minibatch index

epoch = iter / n_minibatches

minibatch_index = iter % n_minibatches

# get the minibatches corresponding to `iter` modulo

# `len(train_batches)`

x,y = train_batches[ minibatch_index ]

if iter %100 == 0:

print 'training @ iter = ', iter

cost_ij = train_model(x,y)

if (iter+1) % validation_frequency == 0:

# compute zero-one loss on validation set

this_validation_loss = 0.

for x,y in valid_batches:

# sum up the errors for each minibatch

this_validation_loss += test_model(x,y)

# get the average by dividing with the number of minibatches

this_validation_loss /= len(valid_batches)

print('epoch %i, minibatch %i/%i, validation error %f %%' % \

(epoch, minibatch_index+1, n_minibatches, \

this_validation_loss*100.))

# if we got the best validation score until now

if this_validation_loss < best_validation_loss:

#improve patience if loss improvement is good enough

if this_validation_loss < best_validation_loss * \

improvement_threshold :

patience = max(patience, iter * patience_increase)

# save best validation score and iteration number

best_validation_loss = this_validation_loss

best_iter = iter

# test it on the test set

test_score = 0.

for x,y in test_batches:

test_score += test_model(x,y)

test_score /= len(test_batches)

print((' epoch %i, minibatch %i/%i, test error of best '

'model %f %%') %

(epoch, minibatch_index+1, n_minibatches,

test_score*100.))

if patience <= iter :

break

end_time = time.clock()

print('Optimization complete.')

print('Best validation score of %f %% obtained at iteration %i,'\

'with test performance %f %%' %

(best_validation_loss * 100., best_iter, test_score*100.))