Repository navigation
Expand file tree
/
Copy patheviction_manager.py
More file actions
47 lines (40 loc) · 1.49 KB
/
Copy patheviction_manager.py
File metadata and controls
47 lines (40 loc) · 1.49 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
# -*- coding: utf-8 -*-
class EvictionManager:
"""
EvictionManager to manager the eviction policy.
:param scalar_storage: CacheStorage to manager the scalar data.
:type scalar_storage: :class:`CacheStorage`
:param vector_base: VectorBase to manager the vector data.
:type vector_base: :class:`VectorBase`
"""
MAX_MARK_COUNT = 5000
MAX_MARK_RATE = 0.1
BATCH_SIZE = 100000
REBUILD_CONDITION = 5
def __init__(self, scalar_storage, vector_base):
self._scalar_storage = scalar_storage
self._vector_base = vector_base
self.delete_count = 0
def check_evict(self):
mark_count = self._scalar_storage.count(state=-1)
all_count = self._scalar_storage.count(is_all=True)
if (
mark_count > self.MAX_MARK_COUNT
or mark_count / all_count > self.MAX_MARK_RATE
):
return True
return False
def delete(self,model):
mark_ids = self._scalar_storage.get_ids(deleted=True)
self._scalar_storage.clear_deleted_data()
self._vector_base.delete(mark_ids,model)
self.delete_count += 1
if self.delete_count >= self.REBUILD_CONDITION:
self.rebuild()
def rebuild(self):
self._scalar_storage.clear_deleted_data()
ids = self._scalar_storage.get_ids(deleted=False)
self._vector_base.rebuild(ids)
self.delete_count = 0
def soft_evict(self, marked_keys):
self._scalar_storage.mark_deleted(marked_keys)