-
Notifications
You must be signed in to change notification settings - Fork 574
Expand file tree
/
Copy pathrandomtrackingdict.py
More file actions
164 lines (139 loc) · 5.24 KB
/
Copy pathrandomtrackingdict.py
File metadata and controls
164 lines (139 loc) · 5.24 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
"""
src/randomtrackingdict.py
=========================
"""
import random
from threading import RLock
from time import time
import helper_random
class RandomTrackingDict(object):
"""
Dict with randomised order and tracking.
Keeps a track of how many items have been requested from the dict, and timeouts. Resets after all objects have been
retrieved and timed out. The main purpose of this isn't as much putting related code together as performance
optimisation and anonymisation of downloading of objects from other peers. If done using a standard dict or array,
it takes too much CPU (and looks convoluted). Randomisation helps with anonymity.
"""
# pylint: disable=too-many-instance-attributes
maxPending = 10
pendingTimeout = 60
def __init__(self):
self.dictionary = {}
self.indexDict = []
self.len = 0
self.pendingLen = 0
self.lastPoll = 0
self.lastObject = 0
self.lock = RLock()
def __len__(self):
return self.len
def __contains__(self, key):
return key in self.dictionary
def __getitem__(self, key):
return self.dictionary[key][1]
def _swap(self, i1, i2):
with self.lock:
key1 = self.indexDict[i1]
key2 = self.indexDict[i2]
self.indexDict[i1] = key2
self.indexDict[i2] = key1
self.dictionary[key1][0] = i2
self.dictionary[key2][0] = i1
# for quick reassignment
return i2
def __setitem__(self, key, value):
with self.lock:
if key in self.dictionary:
self.dictionary[key][1] = value
else:
self.indexDict.append(key)
self.dictionary[key] = [self.len, value]
self._swap(self.len, self.len - self.pendingLen)
self.len += 1
def __delitem__(self, key):
if key not in self.dictionary:
raise KeyError
with self.lock:
index = self.dictionary[key][0]
# not pending
if index < self.len - self.pendingLen:
# left of pending part
index = self._swap(index, self.len - self.pendingLen - 1)
# pending
else:
self.pendingLen -= 1
# end
self._swap(index, self.len - 1)
# if the following del is batched, performance of this single
# operation can improve 4x, but it's already very fast so we'll
# ignore it for the time being
del self.indexDict[-1]
del self.dictionary[key]
self.len -= 1
def setMaxPending(self, maxPending):
"""
Sets maximum number of objects that can be retrieved from the class simultaneously as long as there is no
timeout
"""
self.maxPending = maxPending
def setPendingTimeout(self, pendingTimeout):
"""Sets how long to wait for a timeout if max pending is reached (or all objects have been retrieved)"""
self.pendingTimeout = pendingTimeout
def setLastObject(self):
"""Update timestamp for tracking of received objects"""
self.lastObject = time()
def randomKeys(self, count=1):
"""Retrieve count random keys from the dict that haven't already been retrieved"""
if self.len == 0 or ((self.pendingLen >= self.maxPending or
self.pendingLen == self.len) and self.lastPoll +
self.pendingTimeout > time()):
raise KeyError
# pylint: disable=redefined-outer-name
with self.lock:
# reset if we've requested all
# and if last object received too long time ago
if self.pendingLen == self.len and self.lastObject + self.pendingTimeout < time():
self.pendingLen = 0
self.setLastObject()
available = self.len - self.pendingLen
if count > available:
count = available
randomIndex = helper_random.randomsample(range(self.len - self.pendingLen), count)
retval = [self.indexDict[i] for i in randomIndex]
for i in sorted(randomIndex, reverse=True):
# swap with one below lowest pending
self._swap(i, self.len - self.pendingLen - 1)
self.pendingLen += 1
self.lastPoll = time()
return retval
if __name__ == '__main__':
# pylint: disable=redefined-outer-name
def randString():
"""helper function for tests, generates a random string"""
retval = b''
for _ in range(32):
retval += chr(random.randint(0, 255))
return retval
a = []
k = RandomTrackingDict()
d = {}
print "populating random tracking dict"
a.append(time())
for i in range(50000):
k[randString()] = True
a.append(time())
print "done"
while k:
retval = k.randomKeys(1000)
if not retval:
print "error getting random keys"
try:
k.randomKeys(100)
print "bad"
except KeyError:
pass
for i in retval:
del k[i]
a.append(time())
for x in range(len(a) - 1):
print "%i: %.3f" % (x, a[x + 1] - a[x])