Skip to content

Commit f6d1bf3

Browse files
kastnerkylekastnerkyle
committed
Merge pull request #86 from sklearn-theano/vgg_caffemodels
Added vgg caffemodel support, though no object for real use just yet
2 parents b46e3ac + 4761f0d commit f6d1bf3

File tree

5 files changed

+260
-25
lines changed

5 files changed

+260
-25
lines changed

‎sklearn_theano/datasets/base.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,21 @@ def get_dataset_dir(dataset_name, data_dir=None, folder=None, create_dir=True):
2626
return data_dir
2727

2828

29-
def download(url, server_fname, local_fname=None, progress_update_percentage=5):
29+
def download(url, server_fname, local_fname=None, progress_update_percentage=5,
30+
bypass_certificate_check=False):
3031
"""
3132
An internet download utility modified from
3233
http://stackoverflow.com/questions/22676/
3334
how-do-i-download-a-file-over-http-using-python/22776#22776
3435
"""
35-
u = urllib.urlopen(url)
36+
if bypass_certificate_check:
37+
import ssl
38+
ctx = ssl.create_default_context()
39+
ctx.check_hostname = False
40+
ctx.verify_mode = ssl.CERT_NONE
41+
u = urllib.urlopen(url, context=ctx)
42+
else:
43+
u = urllib.urlopen(url)
3644
if local_fname is None:
3745
local_fname = server_fname
3846
full_path = local_fname

‎sklearn_theano/feature_extraction/caffe/balanced_vgg.py‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
"""Parser for balanced VGG caffe."""
2+
# Authors: Michael Eickenberg
3+
# Kyle Kastner
4+
# License: BSD 3 Clause
5+
6+
from sklearn.externals import joblib
7+
from ...datasets import get_dataset_dir, download
8+
from .caffemodel import _parse_caffe_model, parse_caffe_model
9+
import os
10+
import theano
11+
12+
BALANCED_VGG_PATH = get_dataset_dir("caffe/balanced_vgg")
13+
14+
15+
def fetch_balanced_vgg_protobuffer_file(caffemodel_file=None):
16+
"""Checks for existence of caffemodel protobuffer.
17+
Downloads it if it cannot be found."""
18+
19+
default_filename = os.path.join(BALANCED_VGG_PATH,
20+
"vgg_normalised.caffemodel")
21+
22+
if caffemodel_file is not None:
23+
if os.path.exists(caffemodel_file):
24+
return caffemodel_file
25+
else:
26+
if os.path.exists(default_filename):
27+
import warnings
28+
warnings.warn('Did not find %s, but found and returned %s.' %
29+
(caffemodel_file, default_filename))
30+
return default_filename
31+
else:
32+
if os.path.exists(default_filename):
33+
return default_filename
34+
35+
# We didn't find the file, let's download it. To the specified location
36+
# if specified, otherwise to the default place
37+
if caffemodel_file is None:
38+
caffemodel_file = default_filename
39+
if not os.path.exists(BALANCED_VGG_PATH):
40+
os.makedirs(BALANCED_VGG_PATH)
41+
42+
url = "https://bethgelab.org/media/uploads/deeptextures/"
43+
url += "vgg_normalised.caffemodel"
44+
# Need to bypass cert for bethge lab download
45+
download(url, caffemodel_file, progress_update_percentage=1,
46+
bypass_certificate_check=True)
47+
return caffemodel_file
48+
49+
50+
def fetch_balanced_vgg_architecture(caffemodel_parsed=None,
51+
caffemodel_protobuffer=None):
52+
"""Fetch a pickled version of the caffe model, represented as list of
53+
dictionaries."""
54+
55+
default_filename = os.path.join(BALANCED_VGG_PATH, 'balanced_vgg.pickle')
56+
if caffemodel_parsed is not None:
57+
if os.path.exists(caffemodel_parsed):
58+
return joblib.load(caffemodel_parsed)
59+
else:
60+
if os.path.exists(default_filename):
61+
import warnings
62+
warnings.warn('Did not find %s, but found %s. Loading it.' %
63+
(caffemodel_parsed, default_filename))
64+
return joblib.load(default_filename)
65+
else:
66+
if os.path.exists(default_filename):
67+
return joblib.load(default_filename)
68+
69+
# We didn't find the file: let's create it by parsing the protobuffer
70+
protobuf_file = fetch_balanced_vgg_protobuffer_file(caffemodel_protobuffer)
71+
model = _parse_caffe_model(protobuf_file)
72+
73+
if caffemodel_parsed is not None:
74+
joblib.dump(model, caffemodel_parsed)
75+
else:
76+
joblib.dump(model, default_filename)
77+
78+
return model
79+
80+
81+
def create_theano_expressions(model=None, verbose=0):
82+
83+
if model is None:
84+
model = fetch_balanced_vgg_architecture()
85+
86+
layers, blobs, inputs, params = parse_caffe_model(model, verbose=verbose)
87+
data_input = inputs['data']
88+
return blobs, data_input
89+
90+
91+
def _get_fprop(output_layers=('conv5_4',), model=None, verbose=0):
92+
93+
if model is None:
94+
model = fetch_balanced_vgg_architecture(model)
95+
96+
expressions, input_data = create_theano_expressions(model,
97+
verbose=verbose)
98+
to_compile = [expressions[expr] for expr in output_layers]
99+
100+
return theano.function([input_data], to_compile)

‎sklearn_theano/feature_extraction/caffe/caffemodel.py‎

Lines changed: 51 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,18 @@
11
"""Makes .caffemodel files readable for sklearn-theano"""
2+
# Authors: Michael Eickenberg
3+
# Kyle Kastner
4+
# Erfan Noury
5+
# Li Yao
6+
# License: BSD 3 Clause
27
from __future__ import print_function
38
import os
49
import numpy as np
510
from collections import OrderedDict
6-
import theano
711
import theano.tensor as T
812
from ...datasets import get_dataset_dir, download
13+
from sklearn_theano.base import Convolution, Relu, LRN, Feedforward, ZeroPad
14+
from sklearn_theano.base import CaffePool
15+
import warnings
916

1017

1118
def _get_caffe_dir():
@@ -23,8 +30,8 @@ def _get_caffe_dir():
2330

2431

2532
def _compile_caffe_protobuf(caffe_proto=None,
26-
proto_src_dir=None,
27-
python_out_dir=None):
33+
proto_src_dir=None,
34+
python_out_dir=None):
2835
"""Compiles protocol buffer to python_out_dir"""
2936

3037
if caffe_proto is None:
@@ -50,7 +57,7 @@ def _compile_caffe_protobuf(caffe_proto=None,
5057
# "/home/user/caffe/src/caffe/proto/caffe.proto")
5158
else:
5259
caffe_proto = os.path.join(caffe_dir, "src", "caffe", "proto",
53-
"caffe.proto")
60+
"caffe.proto")
5461
if not os.path.exists(caffe_proto):
5562
raise ValueError(
5663
("Could not find {pf}. Please specify the correct"
@@ -135,6 +142,7 @@ def _blob_to_ndarray(blob):
135142
CONCAT=(('concat_param', 'concat_dim'),),
136143
INNER_PRODUCT=('blobs',),
137144
SOFTMAX_LOSS=None,
145+
SOFTMAX=None,
138146
DROPOUT=None
139147
)
140148

@@ -152,7 +160,7 @@ def _get_property(obj, property_path):
152160

153161

154162
def _parse_caffe_model(caffe_model):
155-
163+
warnings.warn("Caching parse for caffemodel, this may take some time")
156164
caffe_pb2 = _get_caffe_pb2() # need to remove this dependence on pb here
157165
try:
158166
_layer_types = caffe_pb2.LayerParameter.LayerType.items()
@@ -167,11 +175,22 @@ def _parse_caffe_model(caffe_model):
167175
if not hasattr(caffe_model, "layers"):
168176
# Consider it a filename
169177
caffe_model = _open_caffe_model(caffe_model)
178+
170179
layers_raw = caffe_model.layers
171180
parsed = []
172-
for layer in layers_raw:
181+
182+
for n, layer in enumerate(layers_raw):
173183
# standard properties
174184
ltype = layer_types[layer.type]
185+
if n == 0 and ltype != 'DATA':
186+
warnings.warn("Caffemodel doesn't start with DATA - adding")
187+
first_layer_descriptor = dict(
188+
type='DATA',
189+
name='data',
190+
top_blobs=('data',),
191+
bottom_blobs=tuple())
192+
parsed.append(first_layer_descriptor)
193+
175194
layer_descriptor = dict(type=ltype,
176195
name=layer.name,
177196
top_blobs=tuple(layer.top),
@@ -192,22 +211,17 @@ def _parse_caffe_model(caffe_model):
192211
return parsed
193212

194213

195-
from sklearn_theano.base import (Convolution, Relu, MaxPool, FancyMaxPool,
196-
LRN, Feedforward, ZeroPad,
197-
CaffePool)
198-
199-
200-
def parse_caffe_model(caffe_model, float_dtype='float32', verbose=0):
201-
214+
def parse_caffe_model(caffe_model, convert_fc_to_conv=True,
215+
float_dtype='float32', verbose=0):
202216
if isinstance(caffe_model, str) or not isinstance(caffe_model, list):
203217
parsed_caffe_model = _parse_caffe_model(caffe_model)
204218
else:
205219
parsed_caffe_model = caffe_model
206220

207-
208221
layers = OrderedDict()
209222
inputs = OrderedDict()
210223
blobs = OrderedDict()
224+
params = OrderedDict()
211225

212226
for i, layer in enumerate(parsed_caffe_model):
213227
layer_type = layer['type']
@@ -267,6 +281,10 @@ def parse_caffe_model(caffe_model, float_dtype='float32', verbose=0):
267281
# ::subsample[1]]
268282

269283
blobs[top_blobs[0]] = expression
284+
285+
params[layer_name + '_conv_W'] = convolution.convolution_filter_
286+
params[layer_name + '_conv_b'] = convolution.biases_
287+
270288
elif layer_type == "RELU":
271289
# RELU layers take input from bottom_blobs, set everything
272290
# negative to zero and write the result to top_blobs
@@ -306,7 +324,7 @@ def parse_caffe_model(caffe_model, float_dtype='float32', verbose=0):
306324
# DROPOUT may figure in some networks, but it is only relevant
307325
# at the learning stage, not at the prediction stage.
308326
pass
309-
elif layer_type == "SOFTMAX_LOSS":
327+
elif layer_type in ["SOFTMAX_LOSS", "SOFTMAX"]:
310328
softmax_input = blobs[bottom_blobs[0]]
311329
# have to write our own softmax expression, because of shape
312330
# issues
@@ -329,7 +347,7 @@ def parse_caffe_model(caffe_model, float_dtype='float32', verbose=0):
329347
lrn_input = blobs[bottom_blobs[0]]
330348
lrn_factor = layer['lrn_param__alpha']
331349
lrn_exponent = layer['lrn_param__beta']
332-
axis = {0:'channels'}[layer['lrn_param__norm_region']]
350+
axis = {0: 'channels'}[layer['lrn_param__norm_region']]
333351
nsize = layer['lrn_param__local_size']
334352
lrn = LRN(nsize, lrn_factor, lrn_exponent, axis=axis)
335353
lrn._build_expression(lrn_input)
@@ -346,15 +364,26 @@ def parse_caffe_model(caffe_model, float_dtype='float32', verbose=0):
346364
weights = layer_blobs[0].astype(float_dtype)
347365
biases = layer_blobs[1].astype(float_dtype).squeeze()
348366
fully_connected_input = blobs[bottom_blobs[0]]
349-
# fc_layer = Feedforward(weights, biases, activation=None)
350-
fc_layer = Convolution(weights.transpose((2, 3, 0, 1)), biases,
351-
activation=None)
367+
if not convert_fc_to_conv:
368+
if fully_connected_input.ndim == 4:
369+
m_, t_, x_, y_ = fully_connected_input.shape
370+
fully_connected_input = fully_connected_input.reshape(
371+
(m_, t_ * x_ * y_))
372+
fc_layer = Feedforward(weights.squeeze().T, biases,
373+
activation=None)
374+
params[layer_name + '_fc_W'] = fc_layer.weights
375+
if fc_layer.biases is not None:
376+
params[layer_name + '_fc_b'] = fc_layer.biases
377+
else:
378+
fc_layer = Convolution(weights.transpose((2, 3, 0, 1)), biases,
379+
activation=None)
380+
params[layer_name + '_conv_W'] = convolution.convolution_filter_
381+
params[layer_name + '_conv_b'] = convolution.biases_
382+
352383
fc_layer._build_expression(fully_connected_input)
353384
layers[layer_name] = fc_layer
354385
blobs[top_blobs[0]] = fc_layer.expression_
355386
else:
356387
raise ValueError('layer type %s is not known to sklearn-theano'
357388
% layer_type)
358-
return layers, blobs, inputs
359-
360-
389+
return layers, blobs, inputs, params

‎sklearn_theano/feature_extraction/caffe/googlenet.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ def create_theano_expressions(model=None, verbose=0):
8686
if model is None:
8787
model = fetch_googlenet_architecture()
8888

89-
layers, blobs, inputs = parse_caffe_model(model, verbose=verbose)
89+
layers, blobs, inputs, params = parse_caffe_model(model, verbose=verbose)
9090
data_input = inputs['data']
9191
return blobs, data_input
9292

0 commit comments

Comments
 (0)