forked from apache/arrow
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathclient.h
More file actions
309 lines (275 loc) · 13 KB
/
Copy pathclient.h
File metadata and controls
309 lines (275 loc) · 13 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <functional>
#include <memory>
#include <string>
#include <vector>
#include "arrow/buffer.h"
#include "arrow/status.h"
#include "arrow/util/macros.h"
#include "arrow/util/visibility.h"
#include "plasma/common.h"
using arrow::Buffer;
using arrow::Status;
namespace plasma {
/// Object buffer data structure.
struct ObjectBuffer {
/// The data buffer.
std::shared_ptr<Buffer> data;
/// The metadata buffer.
std::shared_ptr<Buffer> metadata;
/// The device number.
int device_num;
};
class ARROW_EXPORT PlasmaClient {
public:
PlasmaClient();
~PlasmaClient();
/// Connect to the local plasma store. Return the resulting connection.
///
/// \param store_socket_name The name of the UNIX domain socket to use to
/// connect to the Plasma store.
/// \param manager_socket_name The name of the UNIX domain socket to use to
/// connect to the local Plasma manager. If this is "", then this
/// function will not connect to a manager.
/// Note that plasma manager is no longer supported, this function
/// will return failure if this is not "".
/// \param release_delay Deprecated (not used).
/// \param num_retries number of attempts to connect to IPC socket, default 50
/// \return The return status.
Status Connect(const std::string& store_socket_name,
const std::string& manager_socket_name = "", int release_delay = 0,
int num_retries = -1);
/// Set runtime options for this client.
///
/// \param client_name The name of the client, used in debug messages.
/// \param output_memory_quota The memory quota in bytes for objects created by
/// this client.
Status SetClientOptions(const std::string& client_name, int64_t output_memory_quota);
/// Create an object in the Plasma Store. Any metadata for this object must be
/// be passed in when the object is created.
///
/// \param object_id The ID to use for the newly created object.
/// \param data_size The size in bytes of the space to be allocated for this
/// object's
/// data (this does not include space used for metadata).
/// \param metadata The object's metadata. If there is no metadata, this
/// pointer
/// should be NULL.
/// \param metadata_size The size in bytes of the metadata. If there is no
/// metadata, this should be 0.
/// \param data The address of the newly created object will be written here.
/// \param device_num The number of the device where the object is being
/// created.
/// device_num = 0 corresponds to the host,
/// device_num = 1 corresponds to GPU0,
/// device_num = 2 corresponds to GPU1, etc.
/// \param evict_if_full Whether to evict other objects to make space for
/// this object.
/// \return The return status.
///
/// The returned object must be released once it is done with. It must also
/// be either sealed or aborted.
Status Create(const ObjectID& object_id, int64_t data_size, const uint8_t* metadata,
int64_t metadata_size, std::shared_ptr<Buffer>* data, int device_num = 0,
bool evict_if_full = true);
/// Create and seal an object in the object store. This is an optimization
/// which allows small objects to be created quickly with fewer messages to
/// the store.
///
/// \param object_id The ID of the object to create.
/// \param data The data for the object to create.
/// \param metadata The metadata for the object to create.
/// \param evict_if_full Whether to evict other objects to make space for
/// this object.
/// \return The return status.
Status CreateAndSeal(const ObjectID& object_id, const std::string& data,
const std::string& metadata, bool evict_if_full = true);
/// Create and seal multiple objects in the object store. This is an optimization
/// of CreateAndSeal to eliminate the cost of IPC per object.
///
/// \param object_ids The vector of IDs of the objects to create.
/// \param data The vector of data for the objects to create.
/// \param metadata The vector of metadata for the objects to create.
/// \param evict_if_full Whether to evict other objects to make space for
/// these objects.
/// \return The return status.
Status CreateAndSealBatch(const std::vector<ObjectID>& object_ids,
const std::vector<std::string>& data,
const std::vector<std::string>& metadata,
bool evict_if_full = true);
/// Get some objects from the Plasma Store. This function will block until the
/// objects have all been created and sealed in the Plasma Store or the
/// timeout expires.
///
/// If an object was not retrieved, the corresponding metadata and data
/// fields in the ObjectBuffer structure will evaluate to false.
/// Objects are automatically released by the client when their buffers
/// get out of scope.
///
/// \param object_ids The IDs of the objects to get.
/// \param timeout_ms The amount of time in milliseconds to wait before this
/// request times out. If this value is -1, then no timeout is set.
/// \param[out] object_buffers The object results.
/// \return The return status.
Status Get(const std::vector<ObjectID>& object_ids, int64_t timeout_ms,
std::vector<ObjectBuffer>* object_buffers);
/// Deprecated variant of Get() that doesn't automatically release buffers
/// when they get out of scope.
///
/// \param object_ids The IDs of the objects to get.
/// \param num_objects The number of object IDs to get.
/// \param timeout_ms The amount of time in milliseconds to wait before this
/// request times out. If this value is -1, then no timeout is set.
/// \param object_buffers An array where the results will be stored.
/// \return The return status.
///
/// The caller is responsible for releasing any retrieved objects, but it
/// should not release objects that were not retrieved.
Status Get(const ObjectID* object_ids, int64_t num_objects, int64_t timeout_ms,
ObjectBuffer* object_buffers);
/// Tell Plasma that the client no longer needs the object. This should be
/// called after Get() or Create() when the client is done with the object.
/// After this call, the buffer returned by Get() is no longer valid.
///
/// \param object_id The ID of the object that is no longer needed.
/// \return The return status.
Status Release(const ObjectID& object_id);
/// Check if the object store contains a particular object and the object has
/// been sealed. The result will be stored in has_object.
///
/// @todo: We may want to indicate if the object has been created but not
/// sealed.
///
/// \param object_id The ID of the object whose presence we are checking.
/// \param has_object The function will write true at this address if
/// the object is present and false if it is not present.
/// \return The return status.
Status Contains(const ObjectID& object_id, bool* has_object);
/// List all the objects in the object store.
///
/// This API is experimental and might change in the future.
///
/// \param[out] objects ObjectTable of objects in the store. For each entry
/// in the map, the following fields are available:
/// - metadata_size: Size of the object metadata in bytes
/// - data_size: Size of the object data in bytes
/// - ref_count: Number of clients referencing the object buffer
/// - create_time: Unix timestamp of the object creation
/// - construct_duration: Object creation time in seconds
/// - state: Is the object still being created or already sealed?
/// \return The return status.
Status List(ObjectTable* objects);
/// Abort an unsealed object in the object store. If the abort succeeds, then
/// it will be as if the object was never created at all. The unsealed object
/// must have only a single reference (the one that would have been removed by
/// calling Seal).
///
/// \param object_id The ID of the object to abort.
/// \return The return status.
Status Abort(const ObjectID& object_id);
/// Seal an object in the object store. The object will be immutable after
/// this
/// call.
///
/// \param object_id The ID of the object to seal.
/// \return The return status.
Status Seal(const ObjectID& object_id);
/// Delete an object from the object store. This currently assumes that the
/// object is present, has been sealed and not used by another client. Otherwise,
/// it is a no operation.
///
/// \todo We may want to allow the deletion of objects that are not present or
/// haven't been sealed.
///
/// \param object_id The ID of the object to delete.
/// \return The return status.
Status Delete(const ObjectID& object_id);
/// Delete a list of objects from the object store. This currently assumes that the
/// object is present, has been sealed and not used by another client. Otherwise,
/// it is a no operation.
///
/// \param object_ids The list of IDs of the objects to delete.
/// \return The return status. If all the objects are nonexistent, return OK.
Status Delete(const std::vector<ObjectID>& object_ids);
/// Delete objects until we have freed up num_bytes bytes or there are no more
/// released objects that can be deleted.
///
/// \param num_bytes The number of bytes to try to free up.
/// \param num_bytes_evicted Out parameter for total number of bytes of space
/// retrieved.
/// \return The return status.
Status Evict(int64_t num_bytes, int64_t& num_bytes_evicted);
/// Bump objects up in the LRU cache, i.e. treat them as recently accessed.
/// Objects that do not exist in the store will be ignored.
///
/// \param object_ids The IDs of the objects to bump.
/// \return The return status.
Status Refresh(const std::vector<ObjectID>& object_ids);
/// Compute the hash of an object in the object store.
///
/// \param object_id The ID of the object we want to hash.
/// \param digest A pointer at which to return the hash digest of the object.
/// The pointer must have at least kDigestSize bytes allocated.
/// \return The return status.
Status Hash(const ObjectID& object_id, uint8_t* digest);
/// Subscribe to notifications when objects are sealed in the object store.
/// Whenever an object is sealed, a message will be written to the client
/// socket that is returned by this method.
///
/// \param fd Out parameter for the file descriptor the client should use to
/// read notifications
/// from the object store about sealed objects.
/// \return The return status.
Status Subscribe(int* fd);
/// Receive next object notification for this client if Subscribe has been called.
///
/// \param fd The file descriptor we are reading the notification from.
/// \param object_id Out parameter, the object_id of the object that was sealed.
/// \param data_size Out parameter, the data size of the object that was sealed.
/// \param metadata_size Out parameter, the metadata size of the object that was sealed.
/// \return The return status.
Status GetNotification(int fd, ObjectID* object_id, int64_t* data_size,
int64_t* metadata_size);
Status DecodeNotifications(const uint8_t* buffer, std::vector<ObjectID>* object_ids,
std::vector<int64_t>* data_sizes,
std::vector<int64_t>* metadata_sizes);
/// Disconnect from the local plasma instance, including the local store and
/// manager.
///
/// \return The return status.
Status Disconnect();
/// Get the current debug string from the plasma store server.
///
/// \return The debug string.
std::string DebugString();
/// Get the memory capacity of the store.
///
/// \return Memory capacity of the store in bytes.
int64_t store_capacity();
private:
friend class PlasmaBuffer;
friend class PlasmaMutableBuffer;
FRIEND_TEST(TestPlasmaStore, GetTest);
FRIEND_TEST(TestPlasmaStore, LegacyGetTest);
FRIEND_TEST(TestPlasmaStore, AbortTest);
bool IsInUse(const ObjectID& object_id);
class ARROW_NO_EXPORT Impl;
std::shared_ptr<Impl> impl_;
};
} // namespace plasma