'Program',

'default_startup_program',

'default_main_program',

'program_guard',

'name_scope',

'cuda_places',

'cpu_places',

'cuda_pinned_places',

'in_dygraph_mode',

"""

"""

assert core.is_compiled_with_cuda(), \

"Not compiled with CUDA"

if device_ids is None:

gpus_env = os.getenv("FLAGS_selected_gpus")

if gpus_env:

device_ids = [int(s) for s in gpus_env.split(",")]

else:

device_ids = six.moves.range(core.get_cuda_device_count())

elif not isinstance(device_ids, (list, tuple)):

device_ids = [device_ids]

"""

if device_count is None:

device_count = _cpu_num()

"""

assert core.is_compiled_with_cuda(), \

"Not compiled with CUDA"

if device_count is None:

device_count = _cpu_num()

def __init__(self, name="", parent=None):

self._children = dict()

self._name = name

self._parent = parent

def child(self, prefix):

if prefix not in self._children:

new_child = NameScope(prefix, self)

self._children[prefix] = [new_child]

else:

new_child = NameScope(prefix + "_%d" % len(self._children[prefix]),

self)

self._children[prefix].append(new_child)

return new_child

def parent(self):

return self._parent

def name(self):

"""

# TODO(panyx0718): Only [0-9a-z].

assert prefix, "namescope prefix cannot be empty."

global _name_scope

_name_scope = _name_scope.child(prefix)

yield

global _name_scope

scope = _name_scope

name = ""

while scope:

name = scope.name() + "/" + name

scope = scope.parent()

"""

"""

dtype = np.dtype(np_dtype)

if dtype == np.float32:

return core.VarDesc.VarType.FP32

elif dtype == np.float64:

return core.VarDesc.VarType.FP64

elif dtype == np.float16:

return core.VarDesc.VarType.FP16

elif dtype == np.int32:

return core.VarDesc.VarType.INT32

elif dtype == np.int16:

return core.VarDesc.VarType.INT16

elif dtype == np.int64:

return core.VarDesc.VarType.INT64

elif dtype == np.bool:

return core.VarDesc.VarType.BOOL

elif dtype == np.uint16:

return core.VarDesc.VarType.INT16

elif dtype == np.uint8:

return core.VarDesc.VarType.UINT8

elif dtype == np.int8:

return core.VarDesc.VarType.INT8

else:

"""

if not isinstance(dtype, core.VarDesc.VarType):

dtype = convert_np_dtype_to_dtype_(dtype)

return dtype in [

core.VarDesc.VarType.FP16, core.VarDesc.VarType.FP32,

core.VarDesc.VarType.FP64

"""

error_fields = list()

if not proto.IsInitialized(error_fields) and throw_on_error:

raise ValueError("{0} are not initialized.\nThe message is {1}:\n".

format(error_fields, proto))

"""

def __init__(self,

block,

type=core.VarDesc.VarType.LOD_TENSOR,

name=None,

shape=None,

dtype=None,

lod_level=None,

capacity=None,

persistable=None,

error_clip=None,

stop_gradient=False,

is_data=False,

**kwargs):

self.block = block

if name is None:

name = unique_name.generate('_generated_var')

if dtype is not None:

if not isinstance(dtype, core.VarDesc.VarType):

dtype = convert_np_dtype_to_dtype_(dtype)

if in_dygraph_mode():

# record vars in tracer rather than blocks

self._ivar = kwargs.get("ivar", None)

if not self._ivar:

self._ivar = core.VarBase(

name, dtype if dtype else core.VarDesc.VarType.FP32,

list(shape) if shape else [],

_current_expected_place(), stop_gradient, True

if persistable else False)

if persistable:

_dygraph_tracer().trace_var(name, self)

self.op = None

else:

self.error_clip = error_clip

is_new_var = False

name = cpt.to_text(name)

self.desc = self.block.desc.find_var(cpt.to_bytes(name))

if self.desc is None:

self.desc = self.block.desc.var(cpt.to_bytes(name))

is_new_var = True

if is_new_var:

self.desc.set_type(type)

elif self.desc.type() != type:

raise ValueError(

"Variable {0} has been created before. The "

"previous type is {1}; the new type is {2}. They"

" are not matched".format(self.name, self.desc.type(),

type))

if shape is not None:

if is_new_var:

self.desc.set_shape(shape)

else:

old_shape = self.shape

shape = tuple(shape)

if shape != old_shape:

raise ValueError(

"Variable {0} has been created before. the previous "

"shape is {1}; the new shape is {2}. They are not "

"matched.".format(self.name, old_shape, shape))

if dtype is not None:

if is_new_var:

self.desc.set_dtype(dtype)

else:

old_dtype = self.dtype

if dtype != old_dtype:

raise ValueError(

"Variable {0} has been created before. "

"The previous data type is {1}; the new "

"data type is {2}. They are not "

"matched.".format(self.name, old_dtype, dtype))

if lod_level is not None:

if is_new_var:

self.desc.set_lod_level(lod_level)

else:

if lod_level != self.lod_level:

raise ValueError(

"Variable {0} has been created before. "

"The previous lod_level is {1}; the new "

"lod_level is {2}. They are not "

"matched".format(self.name, self.lod_level,

lod_level))

if persistable is not None:

if is_new_var:

self.desc.set_persistable(persistable)

else:

if persistable != self.persistable:

raise ValueError(

"Variable {0} has been created before."

"The previous persistable is {1}; the new "

"persistable is {2}. They are not matched".format(

self.name, self.persistable, persistable))

if capacity is not None:

if is_new_var:

self.desc.set_capacity(capacity)

else:

# TODO(abhinavarora) : Compare with set capacity once,

# get_capacity is implemented

pass

self.block.vars[name] = self

self.op = None

self._stop_gradient = stop_gradient

self.is_data = is_data

def numpy(self):

new_ivar = self._ivar._copy_to(core.CPUPlace(), True)

return np.array(new_ivar.value().get_tensor())

def backward(self, backward_strategy=None):

from .dygraph import BackwardStrategy

if backward_strategy is None:

backward_strategy = BackwardStrategy()

backward_strategy.sort_sum_gradient = False

self._ivar._run_backward(backward_strategy)

_dygraph_tracer()._clear_ops()

def gradient(self):

new_ivar = self._ivar._grad_ivar()._copy_to(core.CPUPlace(), True)

return np.array(new_ivar.value().get_tensor())

def clear_gradient(self):

self._ivar._clear_gradient()

def __str__(self):

return self.to_string(True)

def to_string(self, throw_on_error, with_details=False):

"""

if in_dygraph_mode():

# TODO(panyx0718): add more dygraph debug info.

return 'name %s, dtype: %s shape: %s %s' % (

self.name, self.dtype, self.shape,

str(self._ivar.value().get_tensor()))

assert isinstance(throw_on_error, bool) and isinstance(with_details,

bool)

protostr = self.desc.serialize_to_string()

proto = framework_pb2.VarDesc.FromString(six.binary_type(protostr))

res_str = _debug_string_(proto, throw_on_error)

if with_details:

additional_attr = ("error_clip", "stop_gradient")

for attr_name in additional_attr:

res_str += "%s: %s\n" % (

attr_name, six.binary_type(getattr(self, attr_name)))

return res_str

__repr__ = __str__

def set_desc(self, input):

"""

self.desc = input

@property

def stop_gradient(self):

if in_dygraph_mode():

return self._ivar.stop_gradient

else:

return self._stop_gradient

@stop_gradient.setter

def stop_gradient(self, s):

if in_dygraph_mode():

self._ivar.stop_gradient = s

else:

self._stop_gradient = s

@property

def persistable(self):

if in_dygraph_mode():

return self._ivar.persistable

else:

return self.desc.persistable()

@persistable.setter

def persistable(self, p):

if in_dygraph_mode():

return self._ivar.persistable

else:

self.desc.set_persistable(p)

@property

def name(self):

if in_dygraph_mode():

return self._ivar.name

else:

return cpt.to_text(self.desc.name())

@name.setter

def name(self, new_name):

if in_dygraph_mode():

self._ivar.name = new_name

else:

self.desc.set_name(new_name)

@property

def shape(self):

# convert to tuple, make it as same as numpy API.

if in_dygraph_mode():

return self._ivar.shape

else:

return tuple(self.desc.shape())

@property

def dtype(self):

if in_dygraph_mode():

return self._ivar.dtype

else:

return self.desc.dtype()

@property

def lod_level(self):

# TODO(minqiyang): Support lod_level in dygraph mode

return self.desc.lod_level()

@property

def type(self):

if in_dygraph_mode():

return self._ivar.dtype

else:

return self.desc.type()

def _set_error_clip(self, error_clip):

"""

self.error_clip = error_clip

def _slice_indices(self, slice, length):

"""

# Compute step and length as integers.

step = 1 if slice.step is None else slice.step

# Raise ValueError for negative length or zero step.

if length < 0:

raise ValueError("length should not be negative")

if step == 0:

raise ValueError("slice step cannot be zero")

# Find lower and upper bounds for start and stop.

lower = -1 if step < 0 else 0

upper = length - 1 if step < 0 else length

# Compute start.

if slice.start is None:

start = upper if step < 0 else lower

else:

start = slice.start

start = max(start + length, lower) if start < 0 else min(start,

upper)

# Compute stop.

if slice.stop is None:

stop = lower if step < 0 else upper

else:

stop = slice.stop

stop = max(stop + length, lower) if stop < 0 else min(stop, upper)

return start, stop, step

def _detectEllipsis(self, item):

has_ellipsis = False

start = 0

end = len(self.shape)

for index, o in enumerate(item):

if o is Ellipsis:

if has_ellipsis:

raise ValueError("Index can have one ellipsis only.")

has_ellipsis = True

start = index

else:

if has_ellipsis:

end = index

return has_ellipsis, start, end

def _reconstructSliceinfo(self, item):

has_ellipsis, start, end = self._detectEllipsis(item)

if has_ellipsis:

newitem = []

for i in range(start):

newitem.append(item[i])

for i in range(start, end):

newitem.append(slice(None, None, None))

for i in range(end, len(item)):

newitem.append(item[i])

return newitem

else:

return None

def _detectContinuesSlice(self, item):

starts = []

ends = []

for index, o in enumerate(item):

if isinstance(o, int):

start = int(o)

if (index > 0 and index >= self.shape[index]) \

or (index < 0 and (index + self.shape[index]) < 0):

raise IndexError("invalid index")

start = max(start + self.shape[index], 0) if start < 0 else min(

start, self.shape[index])

starts.append(start)

ends.append(start + 1)

elif isinstance(o, slice):

start, stop, step = self._slice_indices(o, self.shape[index])

if step == 1 or step == -1:

starts.append(start)

ends.append(stop)

else:

return False, None

else:

raise IndexError("Valid index accept int or slice or ellipsis")

return True, [starts, ends]

def _cloneVar(self, copy=False):

if not copy:

return self.block.create_var(

name=unique_name.generate(".".join(self.name)),

dtype=self.dtype,

persistable=self.persistable,

stop_gradient=self.stop_gradient, )

else:

return self

def _sliceVar(self, axes, starts, ends):

new_var = self._cloneVar()

self.block.append_op(

type="slice",

inputs={'Input': [self]},

outputs={'Out': [new_var]},

attrs={'axes': axes,

'starts': starts,

'ends': ends})

return new_var

def _concatVar(self, inputs, axis):

new_var = self._cloneVar()

self.block.append_op(

type="concat",

inputs={'X': inputs},

outputs={'Out': [new_var]},

attrs={'axis': axis, })

return new_var

def _sliceAndConcatVar(self, item, axis):

if isinstance(item, slice):

if self.shape[axis] < 0:

return self._cloneVar(True)

start, stop, step = self._slice_indices(item, self.shape[axis])

if step == 1:

return self._sliceVar([axis], [start], [stop])

else:

vars = []

if step > 0:

while start < stop:

vars.append(

self._sliceVar([axis], [start], [start + 1]))

start += step

else:

while start > stop:

vars.append(

self._sliceVar([axis], [start], [start + 1]))

start += step

return self._concatVar(vars, axis)

elif isinstance(item, int):

if self.shape[axis] < 0:

return self._cloneVar(True)

index = int(item)

if (index > 0 and index >= self.shape[axis])\

or (index < 0 and (index + self.shape[axis]) < 0):

raise IndexError("invalid index")

return self._sliceVar([axis], [index], [index + 1])

else:

raise IndexError("Valid index accept int or slice or tuple")

def __getitem__(self, item):

"""

new_var = None

if isinstance(item, tuple):

if len(item) > len(self.shape):

raise IndexError("Too many indexes")

fixedSize = True

for i in range(len(self.shape)):

if self.shape[i] == -1:

fixedSize = False

break

newitem = self._reconstructSliceinfo(item) or item

if fixedSize:

check, info = self._detectContinuesSlice(newitem)

if check:

starts = info[0]

ends = info[1]

axes = [i for i in range(len(starts))]

return self._sliceVar(axes, starts, ends)

else:

new_var = self

for index, o in enumerate(newitem):

new_var = new_var._sliceAndConcatVar(o, index)

else:

new_var = self

for index, o in enumerate(newitem):

new_var = new_var._sliceAndConcatVar(o, index)

else:

new_var = self._sliceAndConcatVar(item, 0)

"""

protostrs = core.get_all_op_protos()

ret_values = []

for pbstr in protostrs:

op_proto = framework_pb2.OpProto.FromString(six.binary_type(pbstr))

ret_values.append(op_proto)

"""

@classmethod

def instance(cls):

if not hasattr(cls, '_instance'):

cls._instance = cls()

return cls._instance

def __init__(self):

assert not hasattr(

self.__class__,

'_instance'), 'Please use `instance()` to get OpProtoHolder object!'

op_protos = get_all_op_protos()

self.op_proto_map = {}

for proto in op_protos:

self.op_proto_map[proto.type] = proto

def get_op_proto(self, type):

"""

if type not in self.op_proto_map:

raise ValueError("Operator \"%s\" has not been registered." % type)

return self.op_proto_map[type]

@staticmethod

def generated_op_attr_names():

return {

core.op_proto_and_checker_maker.kOpRoleAttrName(),

core.op_proto_and_checker_maker.kOpRoleVarAttrName(),

core.op_proto_and_checker_maker.kOpNameScopeAttrName(),

core.op_proto_and_checker_maker.kOpCreationCallstackAttrName()

"""

OP_WITHOUT_KERNEL_SET = {

'feed', 'fetch', 'recurrent', 'go', 'rnn_memory_helper_grad',

'conditional_block', 'while', 'send', 'recv', 'listen_and_serv',

'ncclInit', 'select', 'checkpoint_notify', 'gen_nccl_id'

}

def __init__(self,

block,

desc,

type=None,

inputs=None,

outputs=None,

attrs=None):

if in_dygraph_mode():

if type is None:

raise ValueError(

"`type` to initialized an Operator can not be None.")

self.iop = core.OpBase(type)

self.previous_ops = []

self.attrs = attrs if attrs else {}

else:

self.block = block

self.desc = desc

# note: not add self.attrs here:

# https://github.com/PaddlePaddle/Paddle/pull/12583#pullrequestreview-145093173

op_attrs = attrs

if op_attrs is None:

op_attrs = dict()

del attrs

op_maker = core.op_proto_and_checker_maker

if op_maker.kOpRoleAttrName() not in op_attrs: