-
Notifications
You must be signed in to change notification settings - Fork 2
Expand file tree
/
Copy pathclient.py
More file actions
94 lines (77 loc) · 3.16 KB
/
client.py
File metadata and controls
94 lines (77 loc) · 3.16 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
import typing
from typing import Optional, List, Sequence
import os
import httpx
from .base_client import BaseHumanloop, AsyncBaseHumanloop
from .environment import HumanloopEnvironment
from .eval_utils import _run_eval, Dataset, File, Evaluator, EvaluatorCheck
from .prompts.client import PromptsClient
from .evaluations.client import EvaluationsClient
from .prompt_utils import populate_template
class ExtendedEvalsClient(EvaluationsClient):
client: BaseHumanloop
def run(
self,
file: File,
name: Optional[str],
dataset: Dataset,
evaluators: Optional[Sequence[Evaluator]] = None,
# logs: typing.Sequence[dict] | None = None,
workers: int = 4,
) -> List[EvaluatorCheck]:
"""Evaluate your function for a given `Dataset` and set of `Evaluators`.
:param file: the Humanloop file being evaluated, including a function to run over the dataset.
:param name: the name of the Evaluation to run. If it does not exist, a new Evaluation will be created under your File.
:param dataset: the dataset to map your function over to produce the outputs required by the Evaluation.
:param evaluators: define how judgments are provided for this Evaluation.
:param workers: the number of threads to process datapoints using your function concurrently.
:return: per Evaluator checks.
"""
if self.client is None:
raise ValueError("Need Humanloop client defined to run evals")
return _run_eval(
client=self.client,
file=file,
name=name,
dataset=dataset,
evaluators=evaluators,
workers=workers,
)
class ExtendedPromptsClient(PromptsClient):
populate_template = staticmethod(populate_template)
class Humanloop(BaseHumanloop):
"""
See docstring of BaseHumanloop.
This class extends the base client that contains the auto generated SDK functionality with custom evaluation utilities.
"""
def __init__(
self,
*,
base_url: typing.Optional[str] = None,
environment: HumanloopEnvironment = HumanloopEnvironment.DEFAULT,
api_key: typing.Optional[str] = os.getenv("HUMANLOOP_API_KEY"),
timeout: typing.Optional[float] = None,
follow_redirects: typing.Optional[bool] = True,
httpx_client: typing.Optional[httpx.Client] = None,
):
"""See docstring of BaseHumanloop.__init__(...)
This method extends the base client with evaluation utilities.
"""
super().__init__(
base_url=base_url,
environment=environment,
api_key=api_key,
timeout=timeout,
follow_redirects=follow_redirects,
httpx_client=httpx_client,
)
eval_client = ExtendedEvalsClient(client_wrapper=self._client_wrapper)
eval_client.client = self
self.evaluations = eval_client
self.prompts = ExtendedPromptsClient(client_wrapper=self._client_wrapper)
class AsyncHumanloop(AsyncBaseHumanloop):
"""
See docstring of AsyncBaseHumanloop.
TODO: Add custom evaluation utilities for async case.
"""
pass