Repository navigation
Expand file tree
/
Copy pathmilvus.py
More file actions
189 lines (168 loc) · 6.8 KB
/
Copy pathmilvus.py
File metadata and controls
189 lines (168 loc) · 6.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
# -*- coding: utf-8 -*-
import logging
from typing import List
from uuid import uuid4
import numpy as np
from modelcache.utils import import_pymilvus
from modelcache.utils.log import modelcache_log
from modelcache.manager.vector_data.base import VectorBase, VectorData
import_pymilvus()
from pymilvus import ( # pylint: disable=C0413
connections,
utility,
FieldSchema,
DataType,
CollectionSchema,
Collection,
MilvusException,
)
class Milvus(VectorBase):
SEARCH_PARAM = {
"IVF_FLAT": {"metric_type": "L2", "params": {"nprobe": 10}},
"IVF_SQ8": {"metric_type": "L2", "params": {"nprobe": 10}},
"IVF_PQ": {"metric_type": "L2", "params": {"nprobe": 10}},
"HNSW": {"metric_type": "L2", "params": {"ef": 10}},
"RHNSW_FLAT": {"metric_type": "L2", "params": {"ef": 10}},
"RHNSW_SQ": {"metric_type": "L2", "params": {"ef": 10}},
"RHNSW_PQ": {"metric_type": "L2", "params": {"ef": 10}},
"IVF_HNSW": {"metric_type": "L2", "params": {"nprobe": 10, "ef": 10}},
"ANNOY": {"metric_type": "L2", "params": {"search_k": 10}},
"AUTOINDEX": {"metric_type": "L2", "params": {}},
}
def __init__(
self,
host: str = "localhost",
port: str = "19530",
user: str = "",
password: str = "",
secure: bool = False,
collection_name: str = "modelcache",
dimension: int = 0,
top_k: int = 1,
index_params: dict = None,
search_params: dict = None,
local_mode: bool = False,
local_data: str = "./milvus_data"
):
if dimension <= 0:
raise ValueError(
f"invalid `dim` param: {dimension} in the Milvus vector store."
)
self._local_mode = local_mode
self._local_data = local_data
self.dimension = dimension
self.top_k = top_k
self.index_params = index_params
if self._local_mode:
self._create_local(port, local_data)
self._connect(host, port, user, password, secure)
self.collection_name = collection_name
self.search_params = (
search_params or self.SEARCH_PARAM[self.index_params["index_type"]]
)
def _connect(self, host, port, user, password, secure):
try:
i = [
connections.get_connection_addr(x[0])
for x in connections.list_connections()
].index({"host": host, "port": port})
self.alias = connections.list_connections()[i][0]
except ValueError:
# Connect to the Milvus instance using the passed in Environment variables
self.alias = uuid4().hex
connections.connect(
alias=self.alias,
host=host,
port=port,
user=user, # type: ignore
password=password, # type: ignore
secure=secure,
timeout=10
)
def _create_collection(self, collection_name):
if not utility.has_collection(collection_name, using=self.alias):
schema = [
FieldSchema(
name="id",
dtype=DataType.INT64,
is_primary=True,
auto_id=False,
),
FieldSchema(
name="embedding", dtype=DataType.FLOAT_VECTOR, dim=self.dimension
),
]
schema = CollectionSchema(schema)
self.col = Collection(
collection_name,
schema=schema,
consistency_level="Session",
using=self.alias,
)
else:
modelcache_log.warning("The %s collection already exists, and it will be used directly.", collection_name)
self.col = Collection(
collection_name, consistency_level="Session", using=self.alias
)
if len(self.col.indexes) == 0:
try:
modelcache_log.info("Attempting creation of Milvus index.")
self.col.create_index("embedding", index_params=self.index_params)
modelcache_log.info("Creation of Milvus index successful.")
except MilvusException as e:
modelcache_log.warning("Error with building index: %s, and attempting creation of default index.", e)
i_p = {"metric_type": "L2", "index_type": "AUTOINDEX", "params": {}}
self.col.create_index("embedding", index_params=i_p)
self.index_params = i_p
else:
self.index_params = self.col.indexes[0].to_dict()["index_param"]
self.col.load()
def _get_collection(self, collection_name):
self.col = Collection(
collection_name, consistency_level="Session", using=self.alias
)
self.col.load()
def mul_add(self, datas: List[VectorData], model=None):
collection_name_model = self.collection_name + '_' + model
self._create_collection(collection_name_model)
data_array, id_array = map(list, zip(*((data.data, data.id) for data in datas)))
np_data = np.array(data_array).astype("float32")
entities = [id_array, np_data]
self.col.insert(entities)
def search(self, data: np.ndarray, top_k: int = -1, model=None):
if top_k == -1:
top_k = self.top_k
collection_name_model = self.collection_name + '_' + model
self._create_collection(collection_name_model)
search_result = self.col.search(
data=data.reshape(1, -1).tolist(),
anns_field="embedding",
param=self.search_params,
limit=top_k,
)
return list(zip(search_result[0].distances, search_result[0].ids))
def delete(self, ids, model=None):
collection_name_model = self.collection_name + '_' + model
self._get_collection(collection_name_model)
del_ids = ",".join([str(x) for x in ids])
resp = self.col.delete(f"id in [{del_ids}]")
delete_count = resp.delete_count
return delete_count
def rebuild_col(self, model):
collection_name_model = self.collection_name + '_' + model
# if col exist, drop col
if not utility.has_collection(collection_name_model, using=self.alias):
return 'model collection not found, please check!'
utility.drop_collection(collection_name_model, using=self.alias)
try:
self._create_collection(collection_name_model)
except Exception as e:
logging.info('create_collection: {}'.format(e))
def rebuild(self, ids=None): # pylint: disable=unused-argument
self.col.compact()
def flush(self):
self.col.flush(_async=True)
def close(self):
self.flush()
if self._local_mode:
self._server.stop()