Repository navigation
Expand file tree
/
Copy pathqdrant.py
More file actions
653 lines (600 loc) · 26.9 KB
/
Copy pathqdrant.py
File metadata and controls
653 lines (600 loc) · 26.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
# Copyright (c) Microsoft. All rights reserved.
import ast
import logging
import sys
from collections.abc import MutableMapping, Sequence
from copy import deepcopy
from typing import Any, ClassVar, Final, Generic, TypeVar
from pydantic import HttpUrl, SecretStr, ValidationError, model_validator
from qdrant_client.async_qdrant_client import AsyncQdrantClient
from qdrant_client.models import (
Datatype,
Distance,
FieldCondition,
Filter,
Fusion,
FusionQuery,
MatchAny,
MatchValue,
PointStruct,
Prefetch,
QueryResponse,
Range,
ScoredPoint,
VectorParams,
)
from typing_extensions import override
from semantic_kernel.connectors.ai.embedding_generator_base import EmbeddingGeneratorBase
from semantic_kernel.data.vector import (
DistanceFunction,
GetFilteredRecordOptions,
IndexKind,
KernelSearchResults,
SearchType,
TModel,
VectorSearch,
VectorSearchOptions,
VectorSearchResult,
VectorStore,
VectorStoreCollection,
VectorStoreCollectionDefinition,
)
from semantic_kernel.exceptions import (
VectorSearchExecutionException,
VectorStoreInitializationException,
VectorStoreModelValidationError,
VectorStoreOperationException,
)
from semantic_kernel.kernel_pydantic import KernelBaseSettings
from semantic_kernel.kernel_types import OneOrMany
from semantic_kernel.utils.feature_stage_decorator import release_candidate
from semantic_kernel.utils.telemetry.user_agent import APP_INFO, prepend_semantic_kernel_to_user_agent
if sys.version_info >= (3, 12):
from typing import override # pragma: no cover
else:
from typing_extensions import override # pragma: no cover
logger: logging.Logger = logging.getLogger(__name__)
TKey = TypeVar("TKey", bound=str | int)
DISTANCE_FUNCTION_MAP: Final[dict[DistanceFunction, Distance]] = {
DistanceFunction.COSINE_SIMILARITY: Distance.COSINE,
DistanceFunction.DOT_PROD: Distance.DOT,
DistanceFunction.EUCLIDEAN_DISTANCE: Distance.EUCLID,
DistanceFunction.MANHATTAN: Distance.MANHATTAN,
DistanceFunction.DEFAULT: Distance.COSINE,
}
INDEX_KIND_MAP: Final[dict[IndexKind, str]] = {
IndexKind.HNSW: "hnsw",
IndexKind.DEFAULT: "hnsw",
}
TYPE_MAPPER_VECTOR: Final[dict[str, Datatype]] = {
"float": Datatype.FLOAT32,
"int": Datatype.UINT8,
"binary": Datatype.UINT8,
"default": Datatype.FLOAT32,
}
IN_MEMORY_STRING: Final[str] = ":memory:"
@release_candidate
class QdrantSettings(KernelBaseSettings):
"""Qdrant settings currently used by the Qdrant Vector Record Store."""
env_prefix: ClassVar[str] = "QDRANT_"
url: HttpUrl | None = None
api_key: SecretStr | None = None
host: str | None = None
port: int | None = None
grpc_port: int | None = None
path: str | None = None
location: str | None = None
prefer_grpc: bool = False
@model_validator(mode="before")
def validate_settings(cls, values: dict):
"""Validate the settings."""
if (
isinstance(values, dict)
and "url" not in values
and "host" not in values
and "path" not in values
and "location" not in values
):
values["location"] = IN_MEMORY_STRING
return values
def model_dump(self, **kwargs):
"""Dump the model."""
dump = super().model_dump(**kwargs)
if "api_key" in dump:
dump["api_key"] = dump["api_key"].get_secret_value()
if "url" in dump:
dump["url"] = str(dump["url"])
return dump
@release_candidate
class QdrantCollection(
VectorStoreCollection[TKey, TModel],
VectorSearch[TKey, TModel],
Generic[TKey, TModel],
):
"""A QdrantCollection is a memory collection that uses Qdrant as the backend."""
qdrant_client: AsyncQdrantClient
named_vectors: bool
supported_key_types: ClassVar[set[str] | None] = {"str", "int"}
supported_vector_types: ClassVar[set[str] | None] = {"float", "int"}
supported_search_types: ClassVar[set[SearchType]] = {SearchType.VECTOR, SearchType.KEYWORD_HYBRID}
def __init__(
self,
record_type: type[TModel],
definition: VectorStoreCollectionDefinition | None = None,
collection_name: str | None = None,
embedding_generator: EmbeddingGeneratorBase | None = None,
named_vectors: bool = True,
url: str | None = None,
api_key: str | None = None,
host: str | None = None,
port: int | None = None,
grpc_port: int | None = None,
path: str | None = None,
location: str | None = None,
prefer_grpc: bool | None = None,
client: AsyncQdrantClient | None = None,
env_file_path: str | None = None,
env_file_encoding: str | None = None,
**kwargs: Any,
) -> None:
"""Initializes a new instance of the QdrantVectorRecordStore.
When using qdrant client, make sure to supply url and api_key.
When using qdrant server, make sure to supply url or host and optionally port.
When using qdrant local, either supply path to use a persisted qdrant instance
or set location to ":memory:" to use an in-memory qdrant instance.
When nothing is supplied, it defaults to an in-memory qdrant instance.
You can also supply a async qdrant client directly.
Args:
record_type (type[TModel]): The type of the data model.
definition (VectorStoreRecordDefinition): The model fields, optional.
collection_name (str): The name of the collection, optional.
embedding_generator (EmbeddingGeneratorBase): The embedding generator to use, optional.
named_vectors (bool): If true, vectors are stored with name (default: True).
url (str): The URL of the Qdrant server (default: {None}).
api_key (str): The API key for the Qdrant server (default: {None}).
host (str): The host of the Qdrant server (default: {None}).
port (int): The port of the Qdrant server (default: {None}).
grpc_port (int): The gRPC port of the Qdrant server (default: {None}).
path (str): The path of the Qdrant server (default: {None}).
location (str): The location of the Qdrant server (default: {None}).
prefer_grpc (bool): If true, gRPC will be preferred (default: {None}).
client (AsyncQdrantClient): The Qdrant client to use (default: {None}).
env_file_path (str): Use the environment settings file as a fallback to environment variables.
env_file_encoding (str): The encoding of the environment settings file.
**kwargs: Additional keyword arguments passed to the client constructor.
"""
if client:
super().__init__(
record_type=record_type,
definition=definition,
collection_name=collection_name,
qdrant_client=client, # type: ignore
named_vectors=named_vectors, # type: ignore
managed_client=False,
embedding_generator=embedding_generator,
)
return
try:
settings = QdrantSettings(
url=url,
api_key=api_key,
host=host,
port=port,
grpc_port=grpc_port,
path=path,
location=location,
prefer_grpc=prefer_grpc,
env_file_path=env_file_path,
env_file_encoding=env_file_encoding,
)
except ValidationError as ex:
raise VectorStoreInitializationException("Failed to create Qdrant settings.", ex) from ex
if APP_INFO:
kwargs.setdefault("metadata", {})
kwargs["metadata"] = prepend_semantic_kernel_to_user_agent(kwargs["metadata"])
try:
client = AsyncQdrantClient(**settings.model_dump(exclude_none=True), **kwargs)
except ValueError as ex:
raise VectorStoreInitializationException("Failed to create Qdrant client.", ex) from ex
super().__init__(
record_type=record_type,
definition=definition,
collection_name=collection_name,
qdrant_client=client,
named_vectors=named_vectors,
embedding_generator=embedding_generator,
)
@override
async def _inner_upsert(
self,
records: Sequence[PointStruct],
**kwargs: Any,
) -> Sequence[TKey]:
await self.qdrant_client.upsert(
collection_name=self.collection_name,
points=records,
**kwargs,
)
return [record.id for record in records] # type: ignore
@override
async def _inner_get(
self,
keys: Sequence[TKey] | None = None,
options: GetFilteredRecordOptions | None = None,
**kwargs: Any,
) -> OneOrMany[Any] | None:
if not keys:
if options is not None:
raise NotImplementedError("Get without keys is not yet implemented.")
return None
if "with_vectors" not in kwargs:
kwargs["with_vectors"] = kwargs.pop("include_vectors", True)
return await self.qdrant_client.retrieve(
collection_name=self.collection_name,
ids=keys,
**kwargs,
)
@override
async def _inner_delete(self, keys: Sequence[TKey], **kwargs: Any) -> None:
await self.qdrant_client.delete(
collection_name=self.collection_name,
points_selector=keys,
**kwargs,
)
@override
async def _inner_search(
self,
search_type: SearchType,
options: VectorSearchOptions,
values: Any | None = None,
vector: Sequence[float | int] | None = None,
**kwargs: Any,
) -> KernelSearchResults[VectorSearchResult[TModel]]:
query_vector: tuple[str, Sequence[float | int]] | Sequence[float | int] | None = None
if not vector:
vector = await self._generate_vector_from_values(values, options)
if not vector:
raise VectorSearchExecutionException("Search requires a vector.")
vector_field = self.definition.try_get_vector_field(options.vector_property_name)
if not vector_field:
raise VectorStoreOperationException(
f"Vector field {options.vector_property_name} not found in data model definition."
)
query_vector = (vector_field.storage_name or vector_field.name, vector) if self.named_vectors else vector
filters: Filter | list[Filter] | None = self._build_filter(options.filter) # type: ignore
filter: Filter | None = Filter(must=filters) if filters and isinstance(filters, list) else filters # type: ignore
if search_type == SearchType.VECTOR:
results = await self.qdrant_client.search(
collection_name=self.collection_name,
query_vector=query_vector, # type: ignore
query_filter=filter,
with_vectors=options.include_vectors,
limit=options.top,
offset=options.skip,
**kwargs,
)
else:
# Hybrid search: vector + keywords (RRF fusion)
# 1. Get keywords and text field
if not values:
raise VectorSearchExecutionException("Hybrid search requires non-empty keywords in values.")
if not options.additional_property_name:
raise VectorSearchExecutionException("Hybrid search requires a keyword field name.")
text_field = next(
field
for field in self.definition.fields
if field.name == options.additional_property_name
or field.storage_name == options.additional_property_name
)
if not text_field:
raise VectorStoreOperationException(
f"Keyword field {options.additional_property_name} not found in data model definition."
)
keyword_filter = deepcopy(filter) if filter else Filter()
keyword_sub_filter = Filter(
should=[
FieldCondition(key=text_field.storage_name or text_field.name, match=MatchAny(any=[kw]))
for kw in values
]
)
if isinstance(keyword_filter.must, list):
keyword_filter.must.append(keyword_sub_filter)
elif isinstance(keyword_filter.must, Filter):
keyword_filter.must = Filter(must=[keyword_filter.must, keyword_sub_filter])
else:
keyword_filter.must = keyword_sub_filter
points = await self.qdrant_client.query_points(
collection_name=self.collection_name,
prefetch=[
Prefetch(
query=vector, # type: ignore
using=vector_field.storage_name or vector_field.name,
filter=filter,
limit=options.top,
),
Prefetch(filter=keyword_filter),
],
query=FusionQuery(fusion=Fusion.RRF),
limit=options.top,
offset=options.skip,
with_vectors=options.include_vectors,
**kwargs,
)
results = points.points
return KernelSearchResults(
results=self._get_vector_search_results_from_results(results, options),
total_count=len(results) if options.include_total_count else None,
)
@override
def _get_record_from_result(self, result: ScoredPoint | QueryResponse) -> Any:
return result
@override
def _get_score_from_result(self, result: ScoredPoint) -> float:
return result.score
@override
def _lambda_parser(self, node: ast.AST) -> Any:
# Qdrant filter translation: output a qdrant_client.models.Filter or FieldCondition tree
# Use correct Match subtypes: MatchAny, MatchValue, etc.
# See: https://python-client.qdrant.tech/qdrant_client.http.models.models#qdrant_client.http.models.models.Filter
match node:
case ast.Compare():
if len(node.ops) > 1:
# Chain comparisons (e.g., 1 < x < 3) become AND of each comparison
conditions = []
for idx in range(len(node.ops)):
left = node.left if idx == 0 else node.comparators[idx - 1]
right = node.comparators[idx]
op = node.ops[idx]
conditions.append(self._lambda_parser(ast.Compare(left=left, ops=[op], comparators=[right])))
return Filter(must=conditions)
left = self._lambda_parser(node.left)
right = self._lambda_parser(node.comparators[0])
op = node.ops[0]
match op:
case ast.In():
# IN: left in right (right is a list)
return FieldCondition(key=left, match=MatchAny(any=right))
case ast.NotIn():
# NOT IN: left not in right
return Filter(must_not=[FieldCondition(key=left, match=MatchAny(any=right))])
case ast.Eq():
return FieldCondition(key=left, match=MatchValue(value=right))
case ast.NotEq():
return Filter(must_not=[FieldCondition(key=left, match=MatchValue(value=right))])
case ast.Gt():
return FieldCondition(key=left, range=Range(gt=right))
case ast.GtE():
return FieldCondition(key=left, range=Range(gte=right))
case ast.Lt():
return FieldCondition(key=left, range=Range(lt=right))
case ast.LtE():
return FieldCondition(key=left, range=Range(lte=right))
raise NotImplementedError(f"Unsupported operator: {type(op)}")
case ast.BoolOp():
op = node.op # type: ignore
values = [self._lambda_parser(v) for v in node.values]
if isinstance(op, ast.And):
return Filter(must=values)
if isinstance(op, ast.Or):
return Filter(should=values)
raise NotImplementedError(f"Unsupported BoolOp: {type(op)}")
case ast.UnaryOp():
match node.op:
case ast.Not():
operand = self._lambda_parser(node.operand)
return Filter(must_not=[operand])
case ast.UAdd() | ast.USub() | ast.Invert():
raise NotImplementedError("Unary +, -, ~ are not supported in Qdrant filters.")
case ast.Attribute():
# Only allow attributes that are in the data model
if node.attr not in self.definition.storage_names:
raise VectorStoreOperationException(
f"Field '{node.attr}' not in data model (storage property names are used)."
)
return node.attr
case ast.Name():
# Only allow names that are in the data model
if node.id not in self.definition.storage_names:
raise VectorStoreOperationException(
f"Field '{node.id}' not in data model (storage property names are used)."
)
return node.id
case ast.Constant():
return node.value
case ast.List():
return [self._lambda_parser(elt) for elt in node.elts]
raise NotImplementedError(f"Unsupported AST node: {type(node)}")
@override
def _serialize_dicts_to_store_models(
self,
records: Sequence[dict[str, Any]],
**kwargs: Any,
) -> Sequence[PointStruct]:
return [
PointStruct(
id=record.pop(self._key_field_name),
vector=record.pop(self.definition.vector_field_names[0])
if not self.named_vectors
else {
field.storage_name or field.name: record.pop(field.name) for field in self.definition.vector_fields
},
payload=record,
)
for record in records
]
@override
def _deserialize_store_models_to_dicts(
self,
records: Sequence[PointStruct] | Sequence[ScoredPoint],
**kwargs: Any,
) -> Sequence[dict[str, Any]]:
return [
{
self._key_field_name: record.id,
**(record.payload if record.payload else {}),
**(
{}
if not record.vector
else record.vector
if isinstance(record.vector, dict)
else {self.definition.vector_field_names[0]: record.vector}
),
}
for record in records
]
@override
async def ensure_collection_exists(self, **kwargs) -> None:
"""Create a new collection in Qdrant.
Args:
**kwargs: Additional keyword arguments.
You can supply all keyword arguments supported by the QdrantClient.create_collection method.
This method creates the vectors_config automatically when not supplied, other params are not set.
Collection name will be set to the collection_name property, cannot be overridden.
"""
if "vectors_config" not in kwargs:
if self.named_vectors:
vectors_config: MutableMapping[str, VectorParams] = {}
for field in self.definition.vector_fields:
if field.index_kind not in INDEX_KIND_MAP:
raise VectorStoreOperationException(f"Index kind {field.index_kind} is not supported.")
if field.distance_function not in DISTANCE_FUNCTION_MAP:
raise VectorStoreOperationException(
f"Distance function {field.distance_function} is not supported."
)
vectors_config[field.storage_name or field.name] = VectorParams(
size=field.dimensions,
distance=DISTANCE_FUNCTION_MAP[field.distance_function],
datatype=TYPE_MAPPER_VECTOR[field.type_ or "default"],
)
kwargs["vectors_config"] = vectors_config
else:
vector = self.definition.try_get_vector_field(None)
if not vector:
raise VectorStoreOperationException("Vector field not found in data model definition.")
if vector.distance_function not in DISTANCE_FUNCTION_MAP:
raise VectorStoreOperationException(
f"Distance function {vector.distance_function} is not supported."
)
kwargs["vectors_config"] = VectorParams(
size=vector.dimensions,
distance=DISTANCE_FUNCTION_MAP[vector.distance_function],
datatype=TYPE_MAPPER_VECTOR[vector.type_ or "default"],
)
if "collection_name" not in kwargs:
kwargs["collection_name"] = self.collection_name
await self.qdrant_client.create_collection(**kwargs)
@override
async def collection_exists(self, **kwargs) -> bool:
return await self.qdrant_client.collection_exists(self.collection_name, **kwargs)
@override
async def ensure_collection_deleted(self, **kwargs) -> None:
await self.qdrant_client.delete_collection(self.collection_name, **kwargs)
def _validate_data_model(self):
"""Internal function that should be overloaded by child classes to validate datatypes, etc.
This should take the VectorStoreRecordDefinition from the item_type and validate it against the store.
Checks should include, allowed naming of parameters, allowed data types, allowed vector dimensions.
"""
super()._validate_data_model()
if len(self.definition.vector_field_names) > 1 and not self.named_vectors:
raise VectorStoreModelValidationError("Only one vector field is allowed when not using named vectors.")
@override
async def __aexit__(self, exc_type, exc_value, traceback) -> None:
"""Exit the context manager."""
if self.managed_client:
await self.qdrant_client.close()
@release_candidate
class QdrantStore(VectorStore):
"""A QdrantStore is a memory store that uses Qdrant as the backend."""
qdrant_client: AsyncQdrantClient
def __init__(
self,
url: str | None = None,
api_key: str | None = None,
host: str | None = None,
port: int | None = None,
grpc_port: int | None = None,
path: str | None = None,
location: str | None = None,
prefer_grpc: bool | None = None,
client: AsyncQdrantClient | None = None,
embedding_generator: EmbeddingGeneratorBase | None = None,
env_file_path: str | None = None,
env_file_encoding: str | None = None,
**kwargs: Any,
) -> None:
"""Initializes a new instance of the QdrantVectorRecordStore.
When using qdrant client, make sure to supply url and api_key.
When using qdrant server, make sure to supply url or host and optionally port.
When using qdrant local, either supply path to use a persisted qdrant instance
or set location to ":memory:" to use an in-memory qdrant instance.
When nothing is supplied, it defaults to an in-memory qdrant instance.
You can also supply a async qdrant client directly.
Args:
url: The URL of the Qdrant server (default: {None}).
api_key: The API key for the Qdrant server (default: {None}).
host: The host of the Qdrant server (default: {None}).
port: The port of the Qdrant server (default: {None}).
grpc_port: The gRPC port of the Qdrant server (default: {None}).
path: The path of the Qdrant server (default: {None}).
location: The location of the Qdrant server (default: {None}).
prefer_grpc: If true, gRPC will be preferred (default: {None}).
client: The Qdrant client to use (default: {None}).
embedding_generator: The embedding generator to use (default: {None}).
env_file_path: Use the environment settings file as a fallback to environment variables.
env_file_encoding: The encoding of the environment settings file.
**kwargs: Additional keyword arguments passed to the client constructor.
"""
if client:
super().__init__(
qdrant_client=client, managed_client=False, embedding_generator=embedding_generator, **kwargs
)
return
try:
settings = QdrantSettings(
url=url,
api_key=api_key,
host=host,
port=port,
grpc_port=grpc_port,
path=path,
location=location,
prefer_grpc=prefer_grpc,
env_file_path=env_file_path,
env_file_encoding=env_file_encoding,
)
except ValidationError as ex:
raise VectorStoreInitializationException("Failed to create Qdrant settings.", ex) from ex
if APP_INFO:
kwargs.setdefault("metadata", {})
kwargs["metadata"] = prepend_semantic_kernel_to_user_agent(kwargs["metadata"])
try:
client = AsyncQdrantClient(**settings.model_dump(exclude_none=True), **kwargs)
except ValueError as ex:
raise VectorStoreInitializationException("Failed to create Qdrant client.", ex) from ex
super().__init__(qdrant_client=client, embedding_generator=embedding_generator, **kwargs)
@override
def get_collection(
self,
record_type: type[TModel],
*,
definition: VectorStoreCollectionDefinition | None = None,
collection_name: str | None = None,
embedding_generator: EmbeddingGeneratorBase | None = None,
**kwargs: Any,
) -> QdrantCollection:
return QdrantCollection(
record_type=record_type,
definition=definition,
collection_name=collection_name,
client=self.qdrant_client,
embedding_generator=embedding_generator or self.embedding_generator,
**kwargs,
)
@override
async def list_collection_names(self, **kwargs: Any) -> Sequence[str]:
collections = await self.qdrant_client.get_collections()
return [collection.name for collection in collections.collections]
@override
async def __aexit__(self, exc_type, exc_value, traceback) -> None:
if self.managed_client:
await self.qdrant_client.close()