forked from sightmachine/SimpleCV
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathMFTracker.py
More file actions
491 lines (377 loc) · 14.2 KB
/
Copy pathMFTracker.py
File metadata and controls
491 lines (377 loc) · 14.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
from SimpleCV.base import np, cv, math, time, spsd
from copy import copy
try:
import cv2
except ImportError:
pass
def mfTracker(img, bb, ts, oldimg, **kwargs):
"""
**DESCRIPTION**
(Dev Zone)
Tracking the object surrounded by the bounding box in the given
image using SURF keypoints.
Warning: Use this if you know what you are doing. Better have a
look at Image.track()
**PARAMETERS**
* *img* - Image - Image to be tracked.
* *bb* - tuple - Bounding Box tuple (x, y, w, h)
* *oldimg* - Image - Previous Image
* *ts* - TrackSet - SimpleCV.Features.TrackSet.
Optional PARAMETERS:
numM - Number of points to be tracked in the bounding box
in height direction.
numN - Number of points to be tracked in the bounding box
in width direction.
margin - Margin around the bounding box.
winsize_lk - Optical Flow search window size.
winsize - Size of quadratic area around the point which is compared.
**RETURNS**
SimpleCV.Features.Tracking.MFTracker
**HOW TO USE**
>>> cam = Camera()
>>> ts = []
>>> img = cam.getImage()
>>> bb = (100, 100, 300, 300) # get BB from somewhere
>>> ts = MFTracker(img, bb, ts, img, numM=15, numN=15, winsize=12)
>>> while (some_condition_here):
... img = cam.getImage()
... bb = ts[-1].bb
... prevImg = img
... ts = MFTracker(img, bb, ts, img, numM=15, numN=15, winsize=12)
... ts[-1].drawBB()
... img.show()
This is too much confusing. Better use
Image.track() method.
READ MORE:
Median Flow Tracker:
Media Flow Tracker is the base tracker that is used in OpenTLD. It is based on
Optical Flow. It calculates optical flow of the points in the bounding box from
frame 1 to frame 2 and from frame 2 to frame 1 and using back track error, removes
false positives. As the name suggests, it takes the median of the flow, and eliminates
points.
"""
numM = 10
numN = 10
margin = 5
winsize_ncc = 10
winsize_lk = 4
for key in kwargs:
if key == 'numM':
numM = kwargs[key]
elif key == 'numN':
numN = kwargs[key]
elif key == 'margin':
margin = kwargs[key]
elif key == 'winsize':
winsize_ncc = kwargs[key]
elif key == 'winsize_lk':
winsize_lk = kwargs[key]
oldg = oldimg.getGrayNumpyCv2()
newg = img.getGrayNumpyCv2()
bb = [bb[0], bb[1], bb[0]+bb[2], bb[1]+bb[3]]
bb, shift = fbtrack(oldg, newg, bb, numM, numN, margin, winsize_ncc, winsize_lk)
bb = [bb[0], bb[1], bb[2]-bb[0], bb[3]-bb[1]]
track = MFTrack(img, bb, shift)
return track
def fbtrack(imgI, imgJ, bb, numM=10, numN=10,margin=5,winsize_ncc=10, winsize_lk=4):
"""
**SUMMARY**
(Dev Zone)
Forward-Backward tracking using Lucas-Kanade Tracker
**PARAMETERS**
imgI - Image contain Object with known BoundingBox (Numpy array)
imgJ - Following image (Numpy array)
bb - Bounding box represented through 2 points (x1,y1,x2,y2)
numM - Number of points in height direction.
numN - Number of points in width direction.
margin - margin (in pixel)
winsize_ncc - Size of quadratic area around the point which is compared.
**RETURNS**
newbb - Bounding box of object in track in imgJ
scaleshift - relative scale change of bb
"""
nPoints = numM*numN
sizePointsArray = nPoints*2
#print bb, "passed in fbtrack"
pt = getFilledBBPoints(bb, numM, numN, margin)
fb, ncc, status, ptTracked = lktrack(imgI, imgJ, pt, nPoints, winsize_ncc, winsize_lk)
nlkPoints = sum(status)[0]
startPoints = []
targetPoints = []
fbLKCleaned = [0.0]*nlkPoints
nccLKCleaned = [0.0]*nlkPoints
M = 2
nRealPoints = 0
for i in range(nPoints):
if ptTracked[M*i] is not -1:
startPoints.append((pt[2 * i],pt[2*i+1]))
targetPoints.append((ptTracked[2 * i], ptTracked[2 * i + 1]))
fbLKCleaned[nRealPoints]=fb[i]
nccLKCleaned[nRealPoints]=ncc[i]
nRealPoints+=1
medFb = getMedian(fbLKCleaned)
medNcc = getMedian(nccLKCleaned)
nAfterFbUsage = 0
for i in range(nlkPoints):
if fbLKCleaned[i] <= medFb and nccLKCleaned[i] >= medNcc:
startPoints[nAfterFbUsage] = startPoints[i]
targetPoints[nAfterFbUsage] = targetPoints[i]
nAfterFbUsage+=1
newBB, scaleshift = predictBB(bb, startPoints, targetPoints, nAfterFbUsage)
#print newBB, "fbtrack passing newBB"
return (newBB, scaleshift)
def lktrack(img1, img2, ptsI, nPtsI, winsize_ncc=10, win_size_lk=4, method=cv2.cv.CV_TM_CCOEFF_NORMED):
"""
**SUMMARY**
(Dev Zone)
Lucas-Kanede Tracker with pyramids
**PARAMETERS**
img1 - Previous image or image containing the known bounding box (Numpy array)
img2 - Current image
ptsI - Points to track from the first image
Format ptsI[0] - x1, ptsI[1] - y1, ptsI[2] - x2, ..
nPtsI - Number of points to track from the first image
winsize_ncc - size of the search window at each pyramid level in LK tracker (in int)
method - Paramete specifying the comparison method for normalized cross correlation
(see http://opencv.itseez.com/modules/imgproc/doc/object_detection.html?highlight=matchtemplate#cv2.matchTemplate)
**RETURNS**
fb - forward-backward confidence value. (corresponds to euclidean distance between).
ncc - normCrossCorrelation values
status - Indicates positive tracks. 1 = PosTrack 0 = NegTrack
ptsJ - Calculated Points of second image
"""
template_pt = []
target_pt = []
fb_pt = []
ptsJ = [-1]*len(ptsI)
for i in range(nPtsI):
template_pt.append((ptsI[2*i],ptsI[2*i+1]))
target_pt.append((ptsI[2*i],ptsI[2*i+1]))
fb_pt.append((ptsI[2*i],ptsI[2*i+1]))
template_pt = np.asarray(template_pt,dtype="float32")
target_pt = np.asarray(target_pt,dtype="float32")
fb_pt = np.asarray(fb_pt,dtype="float32")
target_pt, status, track_error = cv2.calcOpticalFlowPyrLK(img1, img2, template_pt, target_pt,
winSize=(win_size_lk, win_size_lk), flags = cv2.OPTFLOW_USE_INITIAL_FLOW,
criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03))
fb_pt, status_bt, track_error_bt = cv2.calcOpticalFlowPyrLK(img2,img1, target_pt,fb_pt,
winSize = (win_size_lk,win_size_lk),flags = cv2.OPTFLOW_USE_INITIAL_FLOW,
criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03))
status = status & status_bt
ncc = normCrossCorrelation(img1, img2, template_pt, target_pt, status, winsize_ncc, method)
fb = euclideanDistance(template_pt, target_pt)
newfb = -1*np.ones(len(fb))
newncc = -1*np.ones(len(ncc))
for i in np.argwhere(status):
i = i[0]
ptsJ[2 * i] = target_pt[i][0]
ptsJ[2 * i + 1] = target_pt[i][1]
newfb[i] = fb[i]
newncc[i] = ncc[i]
return newfb, newncc, status, ptsJ
def getMedianUnmanaged(a):
if not a:
return None
newl = copy(a)
newl.sort()
while True:
try:
newl.remove(0)
except ValueError:
if newl:
return newl[len(newl)/2]
return 0
def getMedian(a):
median = getMedianUnmanaged(a)
return median
def calculateBBCenter(bb):
"""
**SUMMARY**
(Dev Zone)
Calculates the center of the given bounding box
**PARAMETERS**
bb - Bounding Box represented through 2 points (x1,y1,x2,y2)
**RETURNS**
center - A tuple of two floating points
"""
center = (0.5*(bb[0] + bb[2]),0.5*(bb[1]+bb[3]))
return center
def getFilledBBPoints(bb, numM, numN, margin):
"""
**SUMMARY**
(Dev Zone)
Creates numM x numN points grid on Bounding Box
**PARAMETERS**
bb - Bounding Box represented through 2 points (x1,y1,x2,y2)
numM - Number of points in height direction.
numN - Number of points in width direction.
margin - margin (in pixel)
**RETURNS**
pt - A list of points (pt[0] - x1, pt[1] - y1, pt[2] - x2, ..)
"""
pointDim = 2
bb_local = (bb[0] + margin, bb[1] + margin, bb[2] - margin, bb[3] - margin)
if numM == 1 and numN == 1 :
pts = calculateBBCenter(bb_local)
return pts
elif numM > 1 and numN == 1:
divM = numM - 1
divN = 2
spaceM = (bb_local[3]-bb_local[1])/divM
center = calculateBBCenter(bb_local)
pt = [0.0]*(2*numM*numN)
for i in range(numN):
for j in range(numM):
pt[i * numM * pointDim + j * pointDim + 0] = center[0]
pt[i * numM * pointDim + j * pointDim + 1] = bb_local[1] + j * spaceM
return pt
elif numM == 1 and numN > 1:
divM = 2
divN = numN - 1
spaceN = (bb_local[2] - bb_local[0]) / divN
center = calculateBBCenter(bb_local)
pt = [0.0]*((numN-1)*numM*pointDim+numN*pointDim)
for i in range(numN):
for j in range(numN):
pt[i * numM * pointDim + j * pointDim + 0] = bb_local[0] + i * spaceN
pt[i * numM * pointDim + j * pointDim + 1] = center[1]
return pt
elif numM > 1 and numN > 1:
divM = numM - 1
divN = numN - 1
spaceN = (bb_local[2] - bb_local[0]) / divN
spaceM = (bb_local[3] - bb_local[1]) / divM
pt = [0.0]*((numN-1)*numM*pointDim+numM*pointDim)
for i in range(numN):
for j in range(numM):
pt[i * numM * pointDim + j * pointDim + 0] = float(bb_local[0] + i * spaceN)
pt[i * numM * pointDim + j * pointDim + 1] = float(bb_local[1] + j * spaceM)
return pt
def getBBWidth(bb):
"""
**SUMMARY**
(Dev Zone)
Get width of the bounding box
**PARAMETERS**
bb - Bounding Box represented through 2 points (x1,y1,x2,y2)
**RETURNS**
width of the bounding box
"""
return bb[2]-bb[0]+1
def getBBHeight(bb):
"""
**SUMMARY**
(Dev Zone)
Get height of the bounding box
**PARAMETERS**
bb - Bounding Box represented through 2 points (x1,y1,x2,y2)
**RETURNS**
height of the bounding box
"""
return bb[3]-bb[1]+1
def predictBB(bb0, pt0, pt1, nPts):
"""
**SUMMARY**
(Dev Zone)
Calculates the new (moved and resized) Bounding box.
Calculation based on all relative distance changes of all points
to every point. Then the Median of the relative Values is used.
**PARAMETERS**
bb0 - Bounding Box represented through 2 points (x1,y1,x2,y2)
pt0 - Starting Points
pt1 - Target Points
nPts - Total number of points (eg. len(pt0))
**RETURNS**
bb1 - new bounding box
shift - relative scale change of bb0
"""
ofx = []
ofy = []
for i in range(nPts):
ofx.append(pt1[i][0]-pt0[i][0])
ofy.append(pt1[i][1]-pt0[i][1])
dx = getMedianUnmanaged(ofx)
dy = getMedianUnmanaged(ofy)
ofx=ofy=0
lenPdist = nPts * (nPts - 1) / 2
dist0=[]
for i in range(nPts):
for j in range(i+1,nPts):
temp0 = ((pt0[i][0] - pt0[j][0])**2 + (pt0[i][1] - pt0[j][1])**2)**0.5
temp1 = ((pt1[i][0] - pt1[j][0])**2 + (pt1[i][1] - pt1[j][1])**2)**0.5
if temp0 != 0:
dist0.append(float(temp1)/temp0)
else:
dist0.append(1.0)
shift = getMedianUnmanaged(dist0)
if shift is None:
return(bb0, 1.0)
# too much variation in shift is due to some errors
if shift > 1.1 or shift < 0.9:
shift = 1
s0 = 0.5 * (shift - 1) * getBBWidth(bb0)
s1 = 0.5 * (shift - 1) * getBBHeight(bb0)
x1 = bb0[0] - s0 + dx
y1 = bb0[1] - s1 + dy
x2 = bb0[2] + s0 + dx
y2 = bb0[3] + s1 + dy
w = x2-x1
h = y2-y1
#print x1,x2,y1,y2,w,h
if x1 <= 0 or x2 <=0 or y1<=0 or y2 <=0 or w <=20 or h <=20:
x1 = bb0[0]
y1 = bb0[1]
x2 = bb0[2]
y2 = bb0[3]
bb1 = (int(x1),int(y1),int(x2),int(y2))
return (bb1, shift)
def getBB(pt0,pt1):
xmax = np.max((pt0[0],pt1[0]))
xmin = np.min((pt0[0],pt1[0]))
ymax = np.max((pt0[1],pt1[1]))
ymin = np.min((pt0[1],pt1[1]))
return xmin,ymin,xmax,ymax
def getRectFromBB(bb):
return bb[0],bb[1],bb[2]-bb[0],bb[3]-bb[1]
def euclideanDistance(point1,point2):
"""
(Dev Zone)
**SUMMARY**
Calculates eculidean distance between two points
**PARAMETERS**
point1 - vector of points
point2 - vector of points with same length
**RETURNS**
match = returns a vector of eculidean distance
"""
match = ((point1[:,0]-point2[:,0])**2+(point1[:,1]-point2[:,1])**2)**0.5
return match
def normCrossCorrelation(img1, img2, pt0, pt1, status, winsize, method=cv2.cv.CV_TM_CCOEFF_NORMED):
"""
**SUMMARY**
(Dev Zone)
Calculates normalized cross correlation for every point.
**PARAMETERS**
img1 - Image 1.
img2 - Image 2.
pt0 - vector of points of img1
pt1 - vector of points of img2
status - Switch which point pairs should be calculated.
if status[i] == 1 => match[i] is calculated.
else match[i] = 0.0
winsize- Size of quadratic area around the point
which is compared.
method - Specifies the way how image regions are compared. see cv2.matchTemplate
**RETURNS**
match - Output: Array will contain ncc values.
0.0 if not calculated.
"""
nPts = len(pt0)
match = np.zeros(nPts)
for i in np.argwhere(status):
i = i[0]
patch1 = cv2.getRectSubPix(img1,(winsize,winsize),tuple(pt0[i]))
patch2 = cv2.getRectSubPix(img2,(winsize,winsize),tuple(pt1[i]))
match[i] = cv2.matchTemplate(patch1,patch2,method)
return match
from SimpleCV.Tracking import MFTrack