forked from activeloopai/deeplake
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy path_tensorflow.py
More file actions
44 lines (34 loc) · 1.16 KB
/
Copy path_tensorflow.py
File metadata and controls
44 lines (34 loc) · 1.16 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
import numpy as np
try:
import tensorflow as tf
from tensorflow.data import Dataset
except ImportError:
raise ImportError(
"TensorFlow is not installed. Please install tensorflow to use this feature."
)
import deeplake
def _to_tensor_spec(col: deeplake.ColumnDefinition):
dtype = col.dtype.id
if dtype == "text":
dtype = "string"
shape = col.dtype.shape
if not shape:
shape = ()
return tf.TensorSpec(shape=shape, dtype=dtype)
def _from_dataset(ds: deeplake.Dataset):
output_signature = []
column_names = []
for col in ds.schema.columns:
column_names.append(col.name)
output_signature.append(_to_tensor_spec(col))
def generator():
for item in ds:
values = []
for i, col in enumerate(column_names):
value = item[col]
signature = output_signature[i]
if len(signature.shape.dims) == 0 and hasattr(value, "item"):
value = value.item()
values.append(value)
yield tuple(values)
return Dataset.from_generator(generator, output_signature=tuple(output_signature))